2024-11-13 10:38:07,097 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 10:38:07,112 main DEBUG Took 0.012627 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 10:38:07,113 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 10:38:07,113 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 10:38:07,114 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 10:38:07,116 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,125 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 10:38:07,140 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,143 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,144 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,146 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,147 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,148 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,149 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,150 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,151 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,152 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,152 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,153 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,154 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 10:38:07,155 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,156 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 10:38:07,158 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 10:38:07,159 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 10:38:07,162 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 10:38:07,162 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 10:38:07,164 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 10:38:07,164 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 10:38:07,175 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 10:38:07,178 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 10:38:07,180 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 10:38:07,181 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 10:38:07,181 main DEBUG createAppenders(={Console}) 2024-11-13 10:38:07,182 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-13 10:38:07,182 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 10:38:07,183 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-13 10:38:07,183 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 10:38:07,184 main DEBUG OutputStream closed 2024-11-13 10:38:07,184 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 10:38:07,184 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 10:38:07,185 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-13 10:38:07,266 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 10:38:07,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 10:38:07,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 10:38:07,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 10:38:07,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 10:38:07,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 10:38:07,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 10:38:07,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 10:38:07,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 10:38:07,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 10:38:07,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 10:38:07,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 10:38:07,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 10:38:07,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 10:38:07,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 10:38:07,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 10:38:07,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 10:38:07,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 10:38:07,281 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 10:38:07,281 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-13 10:38:07,282 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 10:38:07,283 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-13T10:38:07,534 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee 2024-11-13 10:38:07,537 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 10:38:07,538 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T10:38:07,547 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-13T10:38:07,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T10:38:07,600 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/cluster_deb536b6-0585-3c7e-f24a-dacaede12fac, deleteOnExit=true 2024-11-13T10:38:07,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T10:38:07,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/test.cache.data in system properties and HBase conf 2024-11-13T10:38:07,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T10:38:07,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/hadoop.log.dir in system properties and HBase conf 2024-11-13T10:38:07,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T10:38:07,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T10:38:07,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T10:38:07,722 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T10:38:07,862 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T10:38:07,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T10:38:07,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T10:38:07,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T10:38:07,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T10:38:07,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T10:38:07,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T10:38:07,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T10:38:07,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T10:38:07,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T10:38:07,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/nfs.dump.dir in system properties and HBase conf 2024-11-13T10:38:07,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/java.io.tmpdir in system properties and HBase conf 2024-11-13T10:38:07,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T10:38:07,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T10:38:07,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T10:38:08,935 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T10:38:09,015 INFO [Time-limited test {}] log.Log(170): Logging initialized @2892ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T10:38:09,106 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T10:38:09,186 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T10:38:09,214 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T10:38:09,214 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T10:38:09,216 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T10:38:09,228 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T10:38:09,230 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/hadoop.log.dir/,AVAILABLE} 2024-11-13T10:38:09,231 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T10:38:09,484 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/java.io.tmpdir/jetty-localhost-43811-hadoop-hdfs-3_4_1-tests_jar-_-any-9306908451904871271/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T10:38:09,493 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43811} 2024-11-13T10:38:09,494 INFO [Time-limited test {}] server.Server(415): Started @3371ms 2024-11-13T10:38:10,008 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T10:38:10,017 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T10:38:10,019 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T10:38:10,019 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T10:38:10,019 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T10:38:10,023 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/hadoop.log.dir/,AVAILABLE} 2024-11-13T10:38:10,024 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T10:38:10,177 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/java.io.tmpdir/jetty-localhost-40939-hadoop-hdfs-3_4_1-tests_jar-_-any-2899072880633218855/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T10:38:10,178 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:40939} 2024-11-13T10:38:10,178 INFO [Time-limited test {}] server.Server(415): Started @4055ms 2024-11-13T10:38:10,253 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T10:38:10,817 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/cluster_deb536b6-0585-3c7e-f24a-dacaede12fac/data/data2/current/BP-1227865098-172.17.0.2-1731494288673/current, will proceed with Du for space computation calculation, 2024-11-13T10:38:10,817 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/cluster_deb536b6-0585-3c7e-f24a-dacaede12fac/data/data1/current/BP-1227865098-172.17.0.2-1731494288673/current, will proceed with Du for space computation calculation, 2024-11-13T10:38:10,881 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T10:38:10,967 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6754d2459c3b6712 with lease ID 0x47a941b87405e1d: Processing first storage report for DS-cfc5f758-0daa-4765-8784-f601ae419df9 from datanode DatanodeRegistration(127.0.0.1:40603, datanodeUuid=31cb90c5-5f29-4598-9974-72160ff757af, infoPort=41871, infoSecurePort=0, ipcPort=39541, storageInfo=lv=-57;cid=testClusterID;nsid=2122195666;c=1731494288673) 2024-11-13T10:38:10,969 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6754d2459c3b6712 with lease ID 0x47a941b87405e1d: from storage DS-cfc5f758-0daa-4765-8784-f601ae419df9 node DatanodeRegistration(127.0.0.1:40603, datanodeUuid=31cb90c5-5f29-4598-9974-72160ff757af, infoPort=41871, infoSecurePort=0, ipcPort=39541, storageInfo=lv=-57;cid=testClusterID;nsid=2122195666;c=1731494288673), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T10:38:10,969 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6754d2459c3b6712 with lease ID 0x47a941b87405e1d: Processing first storage report for DS-23c8596c-63d2-477a-9a9f-0a21a18393c4 from datanode DatanodeRegistration(127.0.0.1:40603, datanodeUuid=31cb90c5-5f29-4598-9974-72160ff757af, infoPort=41871, infoSecurePort=0, ipcPort=39541, storageInfo=lv=-57;cid=testClusterID;nsid=2122195666;c=1731494288673) 2024-11-13T10:38:10,970 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6754d2459c3b6712 with lease ID 0x47a941b87405e1d: from storage DS-23c8596c-63d2-477a-9a9f-0a21a18393c4 node DatanodeRegistration(127.0.0.1:40603, datanodeUuid=31cb90c5-5f29-4598-9974-72160ff757af, infoPort=41871, infoSecurePort=0, ipcPort=39541, storageInfo=lv=-57;cid=testClusterID;nsid=2122195666;c=1731494288673), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T10:38:11,032 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee 2024-11-13T10:38:11,135 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/cluster_deb536b6-0585-3c7e-f24a-dacaede12fac/zookeeper_0, clientPort=49884, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/cluster_deb536b6-0585-3c7e-f24a-dacaede12fac/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/cluster_deb536b6-0585-3c7e-f24a-dacaede12fac/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T10:38:11,146 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49884 2024-11-13T10:38:11,160 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T10:38:11,165 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T10:38:11,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741825_1001 (size=7) 2024-11-13T10:38:11,961 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617 with version=8 2024-11-13T10:38:11,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/hbase-staging 2024-11-13T10:38:12,070 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T10:38:12,359 INFO [Time-limited test {}] client.ConnectionUtils(128): master/770665a7984d:0 server-side Connection retries=45 2024-11-13T10:38:12,372 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T10:38:12,373 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T10:38:12,378 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T10:38:12,379 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T10:38:12,379 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T10:38:12,558 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T10:38:12,627 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T10:38:12,637 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T10:38:12,641 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T10:38:12,669 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5503 (auto-detected) 2024-11-13T10:38:12,670 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T10:38:12,691 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44523 2024-11-13T10:38:12,714 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44523 connecting to ZooKeeper ensemble=127.0.0.1:49884 2024-11-13T10:38:12,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:445230x0, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T10:38:12,752 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44523-0x10110e811380000 connected 2024-11-13T10:38:12,793 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T10:38:12,796 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T10:38:12,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T10:38:12,814 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617, hbase.cluster.distributed=false 2024-11-13T10:38:12,845 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T10:38:12,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44523 2024-11-13T10:38:12,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44523 2024-11-13T10:38:12,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44523 2024-11-13T10:38:12,863 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44523 2024-11-13T10:38:12,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44523 2024-11-13T10:38:12,998 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/770665a7984d:0 server-side Connection retries=45 2024-11-13T10:38:13,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T10:38:13,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T10:38:13,000 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T10:38:13,001 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T10:38:13,001 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T10:38:13,004 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T10:38:13,006 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T10:38:13,007 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39101 2024-11-13T10:38:13,009 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39101 connecting to ZooKeeper ensemble=127.0.0.1:49884 2024-11-13T10:38:13,010 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T10:38:13,013 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T10:38:13,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391010x0, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T10:38:13,021 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:391010x0, quorum=127.0.0.1:49884, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T10:38:13,021 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39101-0x10110e811380001 connected 2024-11-13T10:38:13,025 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T10:38:13,040 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T10:38:13,043 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T10:38:13,048 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T10:38:13,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39101 2024-11-13T10:38:13,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39101 2024-11-13T10:38:13,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39101 2024-11-13T10:38:13,051 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39101 2024-11-13T10:38:13,051 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39101 2024-11-13T10:38:13,067 DEBUG [M:0;770665a7984d:44523 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;770665a7984d:44523 2024-11-13T10:38:13,068 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/770665a7984d,44523,1731494292131 2024-11-13T10:38:13,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T10:38:13,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T10:38:13,080 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/770665a7984d,44523,1731494292131 2024-11-13T10:38:13,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:13,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T10:38:13,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:13,115 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T10:38:13,117 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/770665a7984d,44523,1731494292131 from backup master directory 2024-11-13T10:38:13,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/770665a7984d,44523,1731494292131 2024-11-13T10:38:13,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T10:38:13,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T10:38:13,123 WARN [master/770665a7984d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T10:38:13,124 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=770665a7984d,44523,1731494292131 2024-11-13T10:38:13,126 INFO [master/770665a7984d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T10:38:13,128 INFO [master/770665a7984d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T10:38:13,199 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/hbase.id] with ID: 2f1b4553-470e-4e06-bc79-b9c06e16b8f8 2024-11-13T10:38:13,199 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/.tmp/hbase.id 2024-11-13T10:38:13,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741826_1002 (size=42) 2024-11-13T10:38:13,614 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/.tmp/hbase.id]:[hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/hbase.id] 2024-11-13T10:38:13,670 INFO [master/770665a7984d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T10:38:13,675 INFO [master/770665a7984d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T10:38:13,696 INFO [master/770665a7984d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-13T10:38:13,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:13,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:13,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741827_1003 (size=196) 2024-11-13T10:38:14,143 INFO [master/770665a7984d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T10:38:14,145 INFO [master/770665a7984d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T10:38:14,162 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:14,167 INFO [master/770665a7984d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T10:38:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741828_1004 (size=1189) 2024-11-13T10:38:14,616 INFO [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store 2024-11-13T10:38:14,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741829_1005 (size=34) 2024-11-13T10:38:15,041 INFO [master/770665a7984d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T10:38:15,044 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:38:15,045 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T10:38:15,045 INFO [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T10:38:15,046 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T10:38:15,047 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T10:38:15,047 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T10:38:15,047 INFO [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T10:38:15,048 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731494295045Disabling compacts and flushes for region at 1731494295045Disabling writes for close at 1731494295047 (+2 ms)Writing region close event to WAL at 1731494295047Closed at 1731494295047 2024-11-13T10:38:15,051 WARN [master/770665a7984d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/.initializing 2024-11-13T10:38:15,051 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/WALs/770665a7984d,44523,1731494292131 2024-11-13T10:38:15,063 INFO [master/770665a7984d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T10:38:15,080 INFO [master/770665a7984d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=770665a7984d%2C44523%2C1731494292131, suffix=, logDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/WALs/770665a7984d,44523,1731494292131, archiveDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/oldWALs, maxLogs=10 2024-11-13T10:38:15,114 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/WALs/770665a7984d,44523,1731494292131/770665a7984d%2C44523%2C1731494292131.1731494295086, exclude list is [], retry=0 2024-11-13T10:38:15,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40603,DS-cfc5f758-0daa-4765-8784-f601ae419df9,DISK] 2024-11-13T10:38:15,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T10:38:15,173 INFO [master/770665a7984d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/WALs/770665a7984d,44523,1731494292131/770665a7984d%2C44523%2C1731494292131.1731494295086 2024-11-13T10:38:15,174 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41871:41871)] 2024-11-13T10:38:15,174 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:38:15,175 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:38:15,179 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,180 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,219 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,250 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T10:38:15,255 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:15,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T10:38:15,259 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,262 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T10:38:15,263 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:15,264 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:15,264 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,267 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T10:38:15,267 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:15,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:15,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,271 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T10:38:15,271 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:15,272 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:15,273 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,277 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,278 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,283 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,284 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,287 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T10:38:15,291 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T10:38:15,295 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T10:38:15,296 INFO [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59350138, jitterRate=-0.11561402678489685}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T10:38:15,302 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731494295193Initializing all the Stores at 1731494295195 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731494295196 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494295196Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494295197 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494295197Cleaning up temporary data from old regions at 1731494295284 (+87 ms)Region opened successfully at 1731494295301 (+17 ms) 2024-11-13T10:38:15,303 INFO [master/770665a7984d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T10:38:15,341 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6179bd3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=770665a7984d/172.17.0.2:0 2024-11-13T10:38:15,383 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T10:38:15,396 INFO [master/770665a7984d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T10:38:15,396 INFO [master/770665a7984d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T10:38:15,399 INFO [master/770665a7984d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T10:38:15,401 INFO [master/770665a7984d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T10:38:15,406 INFO [master/770665a7984d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T10:38:15,406 INFO [master/770665a7984d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T10:38:15,442 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T10:38:15,454 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T10:38:15,456 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T10:38:15,459 INFO [master/770665a7984d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T10:38:15,461 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T10:38:15,463 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T10:38:15,465 INFO [master/770665a7984d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T10:38:15,469 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T10:38:15,471 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T10:38:15,473 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T10:38:15,477 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T10:38:15,498 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T10:38:15,504 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T10:38:15,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T10:38:15,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:15,513 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=770665a7984d,44523,1731494292131, sessionid=0x10110e811380000, setting cluster-up flag (Was=false) 2024-11-13T10:38:15,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T10:38:15,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:15,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:15,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:15,544 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T10:38:15,547 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=770665a7984d,44523,1731494292131 2024-11-13T10:38:15,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:15,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:15,577 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T10:38:15,585 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=770665a7984d,44523,1731494292131 2024-11-13T10:38:15,598 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T10:38:15,682 INFO [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(746): ClusterId : 2f1b4553-470e-4e06-bc79-b9c06e16b8f8 2024-11-13T10:38:15,686 DEBUG [RS:0;770665a7984d:39101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T10:38:15,693 DEBUG [RS:0;770665a7984d:39101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T10:38:15,693 DEBUG [RS:0;770665a7984d:39101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T10:38:15,699 DEBUG [RS:0;770665a7984d:39101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T10:38:15,699 DEBUG [RS:0;770665a7984d:39101 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@418da01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=770665a7984d/172.17.0.2:0 2024-11-13T10:38:15,726 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T10:38:15,730 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;770665a7984d:39101 2024-11-13T10:38:15,735 INFO [RS:0;770665a7984d:39101 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T10:38:15,736 INFO [RS:0;770665a7984d:39101 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T10:38:15,736 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T10:38:15,741 INFO [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(2659): reportForDuty to master=770665a7984d,44523,1731494292131 with port=39101, startcode=1731494292956 2024-11-13T10:38:15,744 INFO [master/770665a7984d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T10:38:15,757 INFO [master/770665a7984d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T10:38:15,762 DEBUG [RS:0;770665a7984d:39101 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T10:38:15,783 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 770665a7984d,44523,1731494292131 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T10:38:15,804 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/770665a7984d:0, corePoolSize=5, maxPoolSize=5 2024-11-13T10:38:15,805 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/770665a7984d:0, corePoolSize=5, maxPoolSize=5 2024-11-13T10:38:15,805 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/770665a7984d:0, corePoolSize=5, maxPoolSize=5 2024-11-13T10:38:15,805 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/770665a7984d:0, corePoolSize=5, maxPoolSize=5 2024-11-13T10:38:15,806 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/770665a7984d:0, corePoolSize=10, maxPoolSize=10 2024-11-13T10:38:15,806 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:15,806 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/770665a7984d:0, corePoolSize=2, maxPoolSize=2 2024-11-13T10:38:15,807 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:15,846 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T10:38:15,847 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T10:38:15,854 INFO [master/770665a7984d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731494325853 2024-11-13T10:38:15,856 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:15,856 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T10:38:15,858 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T10:38:15,862 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T10:38:15,863 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T10:38:15,863 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T10:38:15,864 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T10:38:15,862 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T10:38:15,877 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:15,909 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T10:38:15,912 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T10:38:15,912 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T10:38:15,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58937, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T10:38:15,932 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44523 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:15,936 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T10:38:15,937 INFO [master/770665a7984d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T10:38:15,961 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/770665a7984d:0:becomeActiveMaster-HFileCleaner.large.0-1731494295939,5,FailOnTimeoutGroup] 2024-11-13T10:38:15,962 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/770665a7984d:0:becomeActiveMaster-HFileCleaner.small.0-1731494295961,5,FailOnTimeoutGroup] 2024-11-13T10:38:15,962 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:15,962 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T10:38:15,967 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:15,968 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:15,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741831_1007 (size=1321) 2024-11-13T10:38:15,994 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T10:38:15,994 WARN [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T10:38:16,098 INFO [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(2659): reportForDuty to master=770665a7984d,44523,1731494292131 with port=39101, startcode=1731494292956 2024-11-13T10:38:16,102 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44523 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 770665a7984d,39101,1731494292956 2024-11-13T10:38:16,106 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44523 {}] master.ServerManager(517): Registering regionserver=770665a7984d,39101,1731494292956 2024-11-13T10:38:16,121 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617 2024-11-13T10:38:16,121 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42945 2024-11-13T10:38:16,121 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T10:38:16,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T10:38:16,147 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [770665a7984d,39101,1731494292956] 2024-11-13T10:38:16,155 DEBUG [RS:0;770665a7984d:39101 {}] zookeeper.ZKUtil(111): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/770665a7984d,39101,1731494292956 2024-11-13T10:38:16,155 WARN [RS:0;770665a7984d:39101 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T10:38:16,155 INFO [RS:0;770665a7984d:39101 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T10:38:16,156 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/WALs/770665a7984d,39101,1731494292956 2024-11-13T10:38:16,253 INFO [RS:0;770665a7984d:39101 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T10:38:16,281 INFO [RS:0;770665a7984d:39101 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T10:38:16,289 INFO [RS:0;770665a7984d:39101 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T10:38:16,290 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:16,305 INFO [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T10:38:16,317 INFO [RS:0;770665a7984d:39101 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T10:38:16,319 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:16,320 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,320 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,320 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,320 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,321 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,321 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/770665a7984d:0, corePoolSize=2, maxPoolSize=2 2024-11-13T10:38:16,321 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,321 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,322 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,322 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,322 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,322 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/770665a7984d:0, corePoolSize=1, maxPoolSize=1 2024-11-13T10:38:16,323 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/770665a7984d:0, corePoolSize=3, maxPoolSize=3 2024-11-13T10:38:16,323 DEBUG [RS:0;770665a7984d:39101 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0, corePoolSize=3, maxPoolSize=3 2024-11-13T10:38:16,359 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:16,360 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:16,360 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:16,360 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:16,360 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:16,360 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=770665a7984d,39101,1731494292956-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T10:38:16,373 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T10:38:16,373 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617 2024-11-13T10:38:16,409 INFO [RS:0;770665a7984d:39101 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T10:38:16,412 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=770665a7984d,39101,1731494292956-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:16,413 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:16,413 INFO [RS:0;770665a7984d:39101 {}] regionserver.Replication(171): 770665a7984d,39101,1731494292956 started 2024-11-13T10:38:16,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741832_1008 (size=32) 2024-11-13T10:38:16,448 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:16,449 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:38:16,451 INFO [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1482): Serving as 770665a7984d,39101,1731494292956, RpcServer on 770665a7984d/172.17.0.2:39101, sessionid=0x10110e811380001 2024-11-13T10:38:16,453 DEBUG [RS:0;770665a7984d:39101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T10:38:16,453 DEBUG [RS:0;770665a7984d:39101 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 770665a7984d,39101,1731494292956 2024-11-13T10:38:16,453 DEBUG [RS:0;770665a7984d:39101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '770665a7984d,39101,1731494292956' 2024-11-13T10:38:16,454 DEBUG [RS:0;770665a7984d:39101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T10:38:16,455 DEBUG [RS:0;770665a7984d:39101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T10:38:16,456 DEBUG [RS:0;770665a7984d:39101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T10:38:16,456 DEBUG [RS:0;770665a7984d:39101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T10:38:16,457 DEBUG [RS:0;770665a7984d:39101 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 770665a7984d,39101,1731494292956 2024-11-13T10:38:16,457 DEBUG [RS:0;770665a7984d:39101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '770665a7984d,39101,1731494292956' 2024-11-13T10:38:16,457 DEBUG [RS:0;770665a7984d:39101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T10:38:16,458 DEBUG [RS:0;770665a7984d:39101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T10:38:16,459 DEBUG [RS:0;770665a7984d:39101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T10:38:16,459 INFO [RS:0;770665a7984d:39101 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T10:38:16,459 INFO [RS:0;770665a7984d:39101 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T10:38:16,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T10:38:16,477 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T10:38:16,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:16,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T10:38:16,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T10:38:16,485 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T10:38:16,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:16,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T10:38:16,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T10:38:16,503 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T10:38:16,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:16,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T10:38:16,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T10:38:16,509 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T10:38:16,509 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:16,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T10:38:16,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T10:38:16,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740 2024-11-13T10:38:16,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740 2024-11-13T10:38:16,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T10:38:16,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T10:38:16,521 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:38:16,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T10:38:16,561 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T10:38:16,566 INFO [RS:0;770665a7984d:39101 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T10:38:16,566 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70314626, jitterRate=0.04776957631111145}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:38:16,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731494296450Initializing all the Stores at 1731494296453 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731494296453Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731494296464 (+11 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494296464Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731494296464Cleaning up temporary data from old regions at 1731494296519 (+55 ms)Region opened successfully at 1731494296570 (+51 ms) 2024-11-13T10:38:16,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T10:38:16,570 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T10:38:16,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T10:38:16,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T10:38:16,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T10:38:16,572 INFO [RS:0;770665a7984d:39101 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=770665a7984d%2C39101%2C1731494292956, suffix=, logDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/WALs/770665a7984d,39101,1731494292956, archiveDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/oldWALs, maxLogs=32 2024-11-13T10:38:16,580 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T10:38:16,581 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731494296570Disabling compacts and flushes for region at 1731494296570Disabling writes for close at 1731494296571 (+1 ms)Writing region close event to WAL at 1731494296575 (+4 ms)Closed at 1731494296575 2024-11-13T10:38:16,589 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T10:38:16,589 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T10:38:16,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T10:38:16,604 DEBUG [RS:0;770665a7984d:39101 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/WALs/770665a7984d,39101,1731494292956/770665a7984d%2C39101%2C1731494292956.1731494296582, exclude list is [], retry=0 2024-11-13T10:38:16,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40603,DS-cfc5f758-0daa-4765-8784-f601ae419df9,DISK] 2024-11-13T10:38:16,619 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T10:38:16,626 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T10:38:16,634 INFO [RS:0;770665a7984d:39101 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/WALs/770665a7984d,39101,1731494292956/770665a7984d%2C39101%2C1731494292956.1731494296582 2024-11-13T10:38:16,636 DEBUG [RS:0;770665a7984d:39101 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41871:41871)] 2024-11-13T10:38:16,784 DEBUG [770665a7984d:44523 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T10:38:16,801 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:38:16,809 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 770665a7984d,39101,1731494292956, state=OPENING 2024-11-13T10:38:16,818 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T10:38:16,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:16,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:38:16,823 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T10:38:16,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T10:38:16,825 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T10:38:16,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:38:17,008 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T10:38:17,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50197, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T10:38:17,025 INFO [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T10:38:17,025 INFO [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T10:38:17,026 INFO [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T10:38:17,034 INFO [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=770665a7984d%2C39101%2C1731494292956.meta, suffix=.meta, logDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/WALs/770665a7984d,39101,1731494292956, archiveDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/oldWALs, maxLogs=32 2024-11-13T10:38:17,054 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/WALs/770665a7984d,39101,1731494292956/770665a7984d%2C39101%2C1731494292956.meta.1731494297036.meta, exclude list is [], retry=0 2024-11-13T10:38:17,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40603,DS-cfc5f758-0daa-4765-8784-f601ae419df9,DISK] 2024-11-13T10:38:17,062 INFO [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/WALs/770665a7984d,39101,1731494292956/770665a7984d%2C39101%2C1731494292956.meta.1731494297036.meta 2024-11-13T10:38:17,063 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41871:41871)] 2024-11-13T10:38:17,063 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:38:17,066 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T10:38:17,069 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T10:38:17,076 INFO [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T10:38:17,081 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T10:38:17,082 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:38:17,083 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T10:38:17,083 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T10:38:17,089 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T10:38:17,091 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T10:38:17,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:17,092 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T10:38:17,092 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T10:38:17,094 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T10:38:17,094 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:17,095 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T10:38:17,096 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T10:38:17,098 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T10:38:17,098 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:17,099 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T10:38:17,099 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T10:38:17,101 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T10:38:17,101 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:17,102 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T10:38:17,102 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T10:38:17,103 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740 2024-11-13T10:38:17,106 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740 2024-11-13T10:38:17,109 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T10:38:17,109 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T10:38:17,111 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:38:17,114 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T10:38:17,116 INFO [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75058086, jitterRate=0.11845263838768005}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:38:17,116 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T10:38:17,118 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731494297083Writing region info on filesystem at 1731494297084 (+1 ms)Initializing all the Stores at 1731494297088 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731494297088Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731494297089 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494297089Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731494297089Cleaning up temporary data from old regions at 1731494297109 (+20 ms)Running coprocessor post-open hooks at 1731494297116 (+7 ms)Region opened successfully at 1731494297118 (+2 ms) 2024-11-13T10:38:17,127 INFO [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731494296996 2024-11-13T10:38:17,141 DEBUG [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T10:38:17,142 INFO [RS_OPEN_META-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T10:38:17,144 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:38:17,146 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 770665a7984d,39101,1731494292956, state=OPEN 2024-11-13T10:38:17,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T10:38:17,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T10:38:17,152 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T10:38:17,153 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T10:38:17,153 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:17,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T10:38:17,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=770665a7984d,39101,1731494292956 in 326 msec 2024-11-13T10:38:17,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T10:38:17,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 561 msec 2024-11-13T10:38:17,170 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T10:38:17,170 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T10:38:17,201 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:17,203 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:17,230 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:17,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45503, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:17,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6220 sec 2024-11-13T10:38:17,273 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731494297273, completionTime=-1 2024-11-13T10:38:17,277 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T10:38:17,277 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T10:38:17,322 INFO [master/770665a7984d:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T10:38:17,322 INFO [master/770665a7984d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731494357322 2024-11-13T10:38:17,322 INFO [master/770665a7984d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731494417322 2024-11-13T10:38:17,322 INFO [master/770665a7984d:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 44 msec 2024-11-13T10:38:17,325 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=770665a7984d,44523,1731494292131-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:17,326 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=770665a7984d,44523,1731494292131-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:17,326 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=770665a7984d,44523,1731494292131-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:17,328 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-770665a7984d:44523, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:17,328 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:17,339 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:17,342 DEBUG [master/770665a7984d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T10:38:17,372 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.247sec 2024-11-13T10:38:17,373 INFO [master/770665a7984d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T10:38:17,375 INFO [master/770665a7984d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T10:38:17,377 INFO [master/770665a7984d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T10:38:17,377 INFO [master/770665a7984d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T10:38:17,377 INFO [master/770665a7984d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T10:38:17,378 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=770665a7984d,44523,1731494292131-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T10:38:17,379 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=770665a7984d,44523,1731494292131-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T10:38:17,401 DEBUG [master/770665a7984d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T10:38:17,402 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T10:38:17,403 INFO [master/770665a7984d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=770665a7984d,44523,1731494292131-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T10:38:17,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132591a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:17,410 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T10:38:17,410 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T10:38:17,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:17,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:17,433 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:17,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672abb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:17,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:17,447 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:17,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c236f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:17,451 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:17,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:17,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=770665a7984d,44523,1731494292131 2024-11-13T10:38:17,516 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=1055, ProcessCount=12, AvailableMemoryMB=1204 2024-11-13T10:38:17,529 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T10:38:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 770665a7984d,44523,1731494292131 2024-11-13T10:38:17,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@485738bd 2024-11-13T10:38:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T10:38:17,547 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T10:38:17,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T10:38:17,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T10:38:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T10:38:17,568 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T10:38:17,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-13T10:38:17,571 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:17,574 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T10:38:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T10:38:17,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741835_1011 (size=959) 2024-11-13T10:38:17,655 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617 2024-11-13T10:38:17,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741836_1012 (size=53) 2024-11-13T10:38:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T10:38:17,705 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:38:17,705 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0704f3508555649d4fe7145badb2caf9, disabling compactions & flushes 2024-11-13T10:38:17,705 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:17,705 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:17,705 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. after waiting 0 ms 2024-11-13T10:38:17,706 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:17,706 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:17,706 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0704f3508555649d4fe7145badb2caf9: Waiting for close lock at 1731494297705Disabling compacts and flushes for region at 1731494297705Disabling writes for close at 1731494297705Writing region close event to WAL at 1731494297706 (+1 ms)Closed at 1731494297706 2024-11-13T10:38:17,720 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T10:38:17,728 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731494297721"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731494297721"}]},"ts":"1731494297721"} 2024-11-13T10:38:17,751 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T10:38:17,762 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T10:38:17,769 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494297762"}]},"ts":"1731494297762"} 2024-11-13T10:38:17,781 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T10:38:17,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0704f3508555649d4fe7145badb2caf9, ASSIGN}] 2024-11-13T10:38:17,791 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0704f3508555649d4fe7145badb2caf9, ASSIGN 2024-11-13T10:38:17,797 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0704f3508555649d4fe7145badb2caf9, ASSIGN; state=OFFLINE, location=770665a7984d,39101,1731494292956; forceNewPlan=false, retain=false 2024-11-13T10:38:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T10:38:17,950 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0704f3508555649d4fe7145badb2caf9, regionState=OPENING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:38:17,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0704f3508555649d4fe7145badb2caf9, ASSIGN because future has completed 2024-11-13T10:38:17,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:38:18,139 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:18,140 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:38:18,140 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,141 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:38:18,141 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,141 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,144 INFO [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,151 INFO [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0704f3508555649d4fe7145badb2caf9 columnFamilyName A 2024-11-13T10:38:18,151 DEBUG [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:18,153 INFO [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] regionserver.HStore(327): Store=0704f3508555649d4fe7145badb2caf9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:18,153 INFO [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,156 INFO [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0704f3508555649d4fe7145badb2caf9 columnFamilyName B 2024-11-13T10:38:18,156 DEBUG [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:18,157 INFO [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] regionserver.HStore(327): Store=0704f3508555649d4fe7145badb2caf9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:18,157 INFO [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,160 INFO [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0704f3508555649d4fe7145badb2caf9 columnFamilyName C 2024-11-13T10:38:18,160 DEBUG [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:18,161 INFO [StoreOpener-0704f3508555649d4fe7145badb2caf9-1 {}] regionserver.HStore(327): Store=0704f3508555649d4fe7145badb2caf9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:18,161 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,163 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,163 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,170 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,170 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,171 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:38:18,177 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,183 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T10:38:18,184 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0704f3508555649d4fe7145badb2caf9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61857533, jitterRate=-0.0782509297132492}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:38:18,184 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:18,185 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0704f3508555649d4fe7145badb2caf9: Running coprocessor pre-open hook at 1731494298141Writing region info on filesystem at 1731494298141Initializing all the Stores at 1731494298143 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494298143Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494298144 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494298144Cleaning up temporary data from old regions at 1731494298170 (+26 ms)Running coprocessor post-open hooks at 1731494298184 (+14 ms)Region opened successfully at 1731494298185 (+1 ms) 2024-11-13T10:38:18,188 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., pid=6, masterSystemTime=1731494298131 2024-11-13T10:38:18,193 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:18,193 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:18,195 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0704f3508555649d4fe7145badb2caf9, regionState=OPEN, openSeqNum=2, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:38:18,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:38:18,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T10:38:18,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 in 234 msec 2024-11-13T10:38:18,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T10:38:18,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0704f3508555649d4fe7145badb2caf9, ASSIGN in 422 msec 2024-11-13T10:38:18,214 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T10:38:18,214 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494298214"}]},"ts":"1731494298214"} 2024-11-13T10:38:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T10:38:18,220 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T10:38:18,223 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T10:38:18,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 664 msec 2024-11-13T10:38:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T10:38:18,729 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:18,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8feb56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:18,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:18,740 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1198e3f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:18,743 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:18,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@207fbc54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,746 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:18,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:18,752 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T10:38:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 770665a7984d,44523,1731494292131 2024-11-13T10:38:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@451b8c3 2024-11-13T10:38:18,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T10:38:18,759 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T10:38:18,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6432e8ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:18,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:18,772 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c85fb36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:18,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:18,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:18,776 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bee8057, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:18,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:18,784 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e11900a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:18,788 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f8ac40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:18,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:18,792 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25175df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:18,799 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:18,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e59e9dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:18,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:18,813 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab7eb50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:18,818 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b025c7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:18,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:18,824 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b41a1cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:18,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:18,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:18,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:18,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72baa874, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ec8db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:18,832 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:18,833 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:18,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b71435d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:18,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:18,837 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:18,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73655f05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:18,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:18,851 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:18,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef2cbdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:18,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:18,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:18,864 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:18,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1beffb12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d81b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:18,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:18,879 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:18,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:18,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dfeb4cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:18,883 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6034ec4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:18,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:18,899 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3334cd5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:18,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eaf4ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:18,913 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:18,913 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:18,917 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:18,917 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:18,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:18,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:38:18,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:38:18,937 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-13T10:38:18,946 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:18,951 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:18,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T10:38:18,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:19,000 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:19,001 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:38:19,015 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:19,019 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:19,026 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:19,029 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:19,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:19,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:19,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:19,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:38:19,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:38:19,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:38:19,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:38:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:38:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:38:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T10:38:19,140 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:19,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:38:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:19,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T10:38:19,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:19,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:19,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T10:38:19,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/8e2b205d2b0d4874b750b7cfc50ebdc2 is 50, key is test_row_0/A:col10/1731494299076/Put/seqid=0 2024-11-13T10:38:19,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741837_1013 (size=12001) 2024-11-13T10:38:19,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/8e2b205d2b0d4874b750b7cfc50ebdc2 2024-11-13T10:38:19,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494359321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494359316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494359321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494359351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494359340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T10:38:19,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:19,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:19,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494359479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494359479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494359479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494359489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494359489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/4e2903e912724ddb9aa0a4bc4e38d17f is 50, key is test_row_0/B:col10/1731494299076/Put/seqid=0 2024-11-13T10:38:19,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T10:38:19,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741838_1014 (size=12001) 2024-11-13T10:38:19,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/4e2903e912724ddb9aa0a4bc4e38d17f 2024-11-13T10:38:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T10:38:19,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:19,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:19,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:19,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/0b9bcebb63f84b1b91f69705cd880524 is 50, key is test_row_0/C:col10/1731494299076/Put/seqid=0 2024-11-13T10:38:19,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494359688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741839_1015 (size=12001) 2024-11-13T10:38:19,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494359698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494359698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494359708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494359709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:19,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/0b9bcebb63f84b1b91f69705cd880524 2024-11-13T10:38:19,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:19,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:19,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/8e2b205d2b0d4874b750b7cfc50ebdc2 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8e2b205d2b0d4874b750b7cfc50ebdc2 2024-11-13T10:38:19,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8e2b205d2b0d4874b750b7cfc50ebdc2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T10:38:19,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T10:38:19,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:19,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:19,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:19,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/4e2903e912724ddb9aa0a4bc4e38d17f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e2903e912724ddb9aa0a4bc4e38d17f 2024-11-13T10:38:19,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e2903e912724ddb9aa0a4bc4e38d17f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T10:38:19,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/0b9bcebb63f84b1b91f69705cd880524 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0b9bcebb63f84b1b91f69705cd880524 2024-11-13T10:38:19,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0b9bcebb63f84b1b91f69705cd880524, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T10:38:19,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0704f3508555649d4fe7145badb2caf9 in 705ms, sequenceid=13, compaction requested=false 2024-11-13T10:38:19,852 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T10:38:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:19,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T10:38:19,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:19,943 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T10:38:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e0c3355dd17e4d96a1507fccf6380a64 is 50, key is test_row_0/A:col10/1731494299320/Put/seqid=0 2024-11-13T10:38:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:20,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741840_1016 (size=12001) 2024-11-13T10:38:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T10:38:20,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494360054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494360075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494360084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494360088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494360096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494360209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494360209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494360209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494360219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494360225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494360427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494360439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494360441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494360441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,450 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e0c3355dd17e4d96a1507fccf6380a64 2024-11-13T10:38:20,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494360449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/5f806a3941734586ba8440c074e6266c is 50, key is test_row_0/B:col10/1731494299320/Put/seqid=0 2024-11-13T10:38:20,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741841_1017 (size=12001) 2024-11-13T10:38:20,609 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/5f806a3941734586ba8440c074e6266c 2024-11-13T10:38:20,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/fbf1873ca7694f6280f6e80ca86cdd85 is 50, key is test_row_0/C:col10/1731494299320/Put/seqid=0 2024-11-13T10:38:20,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741842_1018 (size=12001) 2024-11-13T10:38:20,709 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/fbf1873ca7694f6280f6e80ca86cdd85 2024-11-13T10:38:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e0c3355dd17e4d96a1507fccf6380a64 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0c3355dd17e4d96a1507fccf6380a64 2024-11-13T10:38:20,736 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0c3355dd17e4d96a1507fccf6380a64, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T10:38:20,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/5f806a3941734586ba8440c074e6266c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/5f806a3941734586ba8440c074e6266c 2024-11-13T10:38:20,755 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/5f806a3941734586ba8440c074e6266c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T10:38:20,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/fbf1873ca7694f6280f6e80ca86cdd85 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/fbf1873ca7694f6280f6e80ca86cdd85 2024-11-13T10:38:20,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494360750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494360757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494360761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494360762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494360773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:20,783 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/fbf1873ca7694f6280f6e80ca86cdd85, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T10:38:20,786 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 0704f3508555649d4fe7145badb2caf9 in 842ms, sequenceid=37, compaction requested=false 2024-11-13T10:38:20,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:20,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:20,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T10:38:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T10:38:20,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T10:38:20,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7980 sec 2024-11-13T10:38:20,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8710 sec 2024-11-13T10:38:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T10:38:21,107 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:21,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-13T10:38:21,118 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T10:38:21,120 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:21,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T10:38:21,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T10:38:21,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:21,279 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T10:38:21,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:21,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/4b6e5d15a8d645349672b710925f5fad is 50, key is test_row_0/A:col10/1731494300033/Put/seqid=0 2024-11-13T10:38:21,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741843_1019 (size=12001) 2024-11-13T10:38:21,336 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/4b6e5d15a8d645349672b710925f5fad 2024-11-13T10:38:21,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/19f482a4a2494f71a5f1583006165103 is 50, key is test_row_0/B:col10/1731494300033/Put/seqid=0 2024-11-13T10:38:21,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494361381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494361387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494361388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494361394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494361398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741844_1020 (size=12001) 2024-11-13T10:38:21,425 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/19f482a4a2494f71a5f1583006165103 2024-11-13T10:38:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T10:38:21,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c35b7be81a344f16ad3ec9f15d099868 is 50, key is test_row_0/C:col10/1731494300033/Put/seqid=0 2024-11-13T10:38:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741845_1021 (size=12001) 2024-11-13T10:38:21,497 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c35b7be81a344f16ad3ec9f15d099868 2024-11-13T10:38:21,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494361507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494361512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494361517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494361517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494361509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/4b6e5d15a8d645349672b710925f5fad as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4b6e5d15a8d645349672b710925f5fad 2024-11-13T10:38:21,554 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4b6e5d15a8d645349672b710925f5fad, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T10:38:21,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/19f482a4a2494f71a5f1583006165103 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/19f482a4a2494f71a5f1583006165103 2024-11-13T10:38:21,568 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/19f482a4a2494f71a5f1583006165103, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T10:38:21,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c35b7be81a344f16ad3ec9f15d099868 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c35b7be81a344f16ad3ec9f15d099868 2024-11-13T10:38:21,586 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c35b7be81a344f16ad3ec9f15d099868, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T10:38:21,589 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 0704f3508555649d4fe7145badb2caf9 in 310ms, sequenceid=50, compaction requested=true 2024-11-13T10:38:21,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:21,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:21,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T10:38:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T10:38:21,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T10:38:21,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 472 msec 2024-11-13T10:38:21,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 492 msec 2024-11-13T10:38:21,731 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:21,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:38:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T10:38:21,755 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:21,756 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-13T10:38:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T10:38:21,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/cd2454d0607c4ee6a5953bc5bc796cb5 is 50, key is test_row_0/A:col10/1731494301728/Put/seqid=0 2024-11-13T10:38:21,760 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:21,761 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:21,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:21,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494361763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494361766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494361768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494361775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494361778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741846_1022 (size=14341) 2024-11-13T10:38:21,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/cd2454d0607c4ee6a5953bc5bc796cb5 2024-11-13T10:38:21,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/23d90964509d4f9e888bf9c72292516a is 50, key is test_row_0/B:col10/1731494301728/Put/seqid=0 2024-11-13T10:38:21,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741847_1023 (size=12001) 2024-11-13T10:38:21,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/23d90964509d4f9e888bf9c72292516a 2024-11-13T10:38:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T10:38:21,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494361887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494361887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494361888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494361897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494361897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:21,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/ec675f575be54ae0b792b74911b74d8d is 50, key is test_row_0/C:col10/1731494301728/Put/seqid=0 2024-11-13T10:38:21,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T10:38:21,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:21,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:21,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:21,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:21,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:21,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741848_1024 (size=12001) 2024-11-13T10:38:21,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/ec675f575be54ae0b792b74911b74d8d 2024-11-13T10:38:21,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/cd2454d0607c4ee6a5953bc5bc796cb5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/cd2454d0607c4ee6a5953bc5bc796cb5 2024-11-13T10:38:21,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/cd2454d0607c4ee6a5953bc5bc796cb5, entries=200, sequenceid=75, filesize=14.0 K 2024-11-13T10:38:21,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/23d90964509d4f9e888bf9c72292516a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/23d90964509d4f9e888bf9c72292516a 2024-11-13T10:38:21,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/23d90964509d4f9e888bf9c72292516a, entries=150, sequenceid=75, filesize=11.7 K 2024-11-13T10:38:21,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/ec675f575be54ae0b792b74911b74d8d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ec675f575be54ae0b792b74911b74d8d 2024-11-13T10:38:21,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ec675f575be54ae0b792b74911b74d8d, entries=150, sequenceid=75, filesize=11.7 K 2024-11-13T10:38:21,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0704f3508555649d4fe7145badb2caf9 in 266ms, sequenceid=75, compaction requested=true 2024-11-13T10:38:21,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:22,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:22,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:22,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:22,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:22,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:22,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:22,011 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:22,011 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:22,026 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:22,028 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:22,029 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:22,029 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e2903e912724ddb9aa0a4bc4e38d17f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/5f806a3941734586ba8440c074e6266c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/19f482a4a2494f71a5f1583006165103, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/23d90964509d4f9e888bf9c72292516a] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=46.9 K 2024-11-13T10:38:22,031 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e2903e912724ddb9aa0a4bc4e38d17f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731494299076 2024-11-13T10:38:22,033 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f806a3941734586ba8440c074e6266c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731494299312 2024-11-13T10:38:22,037 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:22,037 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 19f482a4a2494f71a5f1583006165103, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494300033 2024-11-13T10:38:22,037 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:22,037 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:22,038 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8e2b205d2b0d4874b750b7cfc50ebdc2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0c3355dd17e4d96a1507fccf6380a64, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4b6e5d15a8d645349672b710925f5fad, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/cd2454d0607c4ee6a5953bc5bc796cb5] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=49.2 K 2024-11-13T10:38:22,038 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d90964509d4f9e888bf9c72292516a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731494301384 2024-11-13T10:38:22,040 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e2b205d2b0d4874b750b7cfc50ebdc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731494299076 2024-11-13T10:38:22,041 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0c3355dd17e4d96a1507fccf6380a64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731494299312 2024-11-13T10:38:22,045 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b6e5d15a8d645349672b710925f5fad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494300033 2024-11-13T10:38:22,046 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd2454d0607c4ee6a5953bc5bc796cb5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731494301384 2024-11-13T10:38:22,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T10:38:22,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:22,075 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:38:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T10:38:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/181ce2dbe09e4772b8387b95223c77a5 is 50, key is test_row_0/A:col10/1731494301775/Put/seqid=0 2024-11-13T10:38:22,123 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#12 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:22,126 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#13 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:22,126 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/59ef647f86ed4116b7aff394c4a03777 is 50, key is test_row_0/B:col10/1731494301728/Put/seqid=0 2024-11-13T10:38:22,131 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e0edaa5f598e413693a38e29bacb83df is 50, key is test_row_0/A:col10/1731494301728/Put/seqid=0 2024-11-13T10:38:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:22,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:22,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741849_1025 (size=12001) 2024-11-13T10:38:22,194 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/181ce2dbe09e4772b8387b95223c77a5 2024-11-13T10:38:22,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741850_1026 (size=12139) 2024-11-13T10:38:22,218 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/59ef647f86ed4116b7aff394c4a03777 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/59ef647f86ed4116b7aff394c4a03777 2024-11-13T10:38:22,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741851_1027 (size=12139) 2024-11-13T10:38:22,241 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e0edaa5f598e413693a38e29bacb83df as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0edaa5f598e413693a38e29bacb83df 2024-11-13T10:38:22,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/2642146c8a91494bba98a397fd601b4e is 50, key is test_row_0/B:col10/1731494301775/Put/seqid=0 2024-11-13T10:38:22,265 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into e0edaa5f598e413693a38e29bacb83df(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:22,265 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:22,268 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=12, startTime=1731494301999; duration=0sec 2024-11-13T10:38:22,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741852_1028 (size=12001) 2024-11-13T10:38:22,270 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into 59ef647f86ed4116b7aff394c4a03777(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:22,270 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:22,270 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=12, startTime=1731494302010; duration=0sec 2024-11-13T10:38:22,270 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:22,270 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:22,271 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:22,269 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:22,271 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:22,277 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:22,277 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:22,277 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:22,277 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0b9bcebb63f84b1b91f69705cd880524, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/fbf1873ca7694f6280f6e80ca86cdd85, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c35b7be81a344f16ad3ec9f15d099868, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ec675f575be54ae0b792b74911b74d8d] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=46.9 K 2024-11-13T10:38:22,278 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b9bcebb63f84b1b91f69705cd880524, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731494299076 2024-11-13T10:38:22,279 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf1873ca7694f6280f6e80ca86cdd85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731494299312 2024-11-13T10:38:22,280 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting c35b7be81a344f16ad3ec9f15d099868, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494300033 2024-11-13T10:38:22,282 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ec675f575be54ae0b792b74911b74d8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731494301384 2024-11-13T10:38:22,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494362278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494362281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494362285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494362288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494362292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,339 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#16 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:22,346 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/e04feef815244765a6dd58b7ffdd7cf0 is 50, key is test_row_0/C:col10/1731494301728/Put/seqid=0 2024-11-13T10:38:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T10:38:22,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741853_1029 (size=12139) 2024-11-13T10:38:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494362417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494362418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494362418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494362419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494362420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,601 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T10:38:22,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494362639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494362639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494362641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494362641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494362641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,672 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/2642146c8a91494bba98a397fd601b4e 2024-11-13T10:38:22,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/32aef205c25143b9926796ba31505239 is 50, key is test_row_0/C:col10/1731494301775/Put/seqid=0 2024-11-13T10:38:22,736 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T10:38:22,738 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T10:38:22,739 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T10:38:22,742 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T10:38:22,742 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T10:38:22,742 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T10:38:22,742 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T10:38:22,744 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T10:38:22,744 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T10:38:22,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741854_1030 (size=12001) 2024-11-13T10:38:22,750 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/32aef205c25143b9926796ba31505239 2024-11-13T10:38:22,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/181ce2dbe09e4772b8387b95223c77a5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/181ce2dbe09e4772b8387b95223c77a5 2024-11-13T10:38:22,774 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/181ce2dbe09e4772b8387b95223c77a5, entries=150, sequenceid=86, filesize=11.7 K 2024-11-13T10:38:22,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/2642146c8a91494bba98a397fd601b4e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/2642146c8a91494bba98a397fd601b4e 2024-11-13T10:38:22,801 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/2642146c8a91494bba98a397fd601b4e, entries=150, sequenceid=86, filesize=11.7 K 2024-11-13T10:38:22,803 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/e04feef815244765a6dd58b7ffdd7cf0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e04feef815244765a6dd58b7ffdd7cf0 2024-11-13T10:38:22,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/32aef205c25143b9926796ba31505239 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/32aef205c25143b9926796ba31505239 2024-11-13T10:38:22,820 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into e04feef815244765a6dd58b7ffdd7cf0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:22,820 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:22,820 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=12, startTime=1731494302011; duration=0sec 2024-11-13T10:38:22,820 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:22,821 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:22,829 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/32aef205c25143b9926796ba31505239, entries=150, sequenceid=86, filesize=11.7 K 2024-11-13T10:38:22,830 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0704f3508555649d4fe7145badb2caf9 in 756ms, sequenceid=86, compaction requested=false 2024-11-13T10:38:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T10:38:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T10:38:22,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T10:38:22,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0710 sec 2024-11-13T10:38:22,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0820 sec 2024-11-13T10:38:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T10:38:22,902 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:22,905 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-13T10:38:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T10:38:22,908 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:22,910 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:22,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:22,964 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:22,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:38:22,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/f55db0272c46467abb577fc8da72e296 is 50, key is test_row_0/A:col10/1731494302962/Put/seqid=0 2024-11-13T10:38:22,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494362982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494362984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494362987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494362988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494362994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741855_1031 (size=12001) 2024-11-13T10:38:23,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/f55db0272c46467abb577fc8da72e296 2024-11-13T10:38:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T10:38:23,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/de0faed06291471e91639e12fddd8285 is 50, key is test_row_0/B:col10/1731494302962/Put/seqid=0 2024-11-13T10:38:23,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741856_1032 (size=12001) 2024-11-13T10:38:23,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494363097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494363108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494363109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494363109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494363110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:23,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:23,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T10:38:23,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494363309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494363319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494363327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494363327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494363348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/de0faed06291471e91639e12fddd8285 2024-11-13T10:38:23,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/6d0acd039f614939ab650a651f8353da is 50, key is test_row_0/C:col10/1731494302962/Put/seqid=0 2024-11-13T10:38:23,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:23,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:23,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T10:38:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741857_1033 (size=12001) 2024-11-13T10:38:23,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/6d0acd039f614939ab650a651f8353da 2024-11-13T10:38:23,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/f55db0272c46467abb577fc8da72e296 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f55db0272c46467abb577fc8da72e296 2024-11-13T10:38:23,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f55db0272c46467abb577fc8da72e296, entries=150, sequenceid=116, filesize=11.7 K 2024-11-13T10:38:23,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/de0faed06291471e91639e12fddd8285 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/de0faed06291471e91639e12fddd8285 2024-11-13T10:38:23,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/de0faed06291471e91639e12fddd8285, entries=150, sequenceid=116, filesize=11.7 K 2024-11-13T10:38:23,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/6d0acd039f614939ab650a651f8353da as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/6d0acd039f614939ab650a651f8353da 2024-11-13T10:38:23,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494363628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494363628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494363638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494363638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:23,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/6d0acd039f614939ab650a651f8353da, entries=150, sequenceid=116, filesize=11.7 K 2024-11-13T10:38:23,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 0704f3508555649d4fe7145badb2caf9 in 699ms, sequenceid=116, compaction requested=true 2024-11-13T10:38:23,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:23,664 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:23,664 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:23,666 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:23,666 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:23,666 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:23,666 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,667 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:23,667 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,667 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0edaa5f598e413693a38e29bacb83df, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/181ce2dbe09e4772b8387b95223c77a5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f55db0272c46467abb577fc8da72e296] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=35.3 K 2024-11-13T10:38:23,667 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/59ef647f86ed4116b7aff394c4a03777, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/2642146c8a91494bba98a397fd601b4e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/de0faed06291471e91639e12fddd8285] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=35.3 K 2024-11-13T10:38:23,668 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0edaa5f598e413693a38e29bacb83df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731494301384 2024-11-13T10:38:23,668 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 59ef647f86ed4116b7aff394c4a03777, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731494301384 2024-11-13T10:38:23,668 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 181ce2dbe09e4772b8387b95223c77a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731494301736 2024-11-13T10:38:23,669 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2642146c8a91494bba98a397fd601b4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731494301736 2024-11-13T10:38:23,669 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f55db0272c46467abb577fc8da72e296, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731494302281 2024-11-13T10:38:23,670 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting de0faed06291471e91639e12fddd8285, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731494302281 2024-11-13T10:38:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:23,691 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:23,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:38:23,693 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#21 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:23,694 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0170adf13ed6427e9a3eb0470e87f334 is 50, key is test_row_0/B:col10/1731494302962/Put/seqid=0 2024-11-13T10:38:23,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:23,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:23,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,709 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#22 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,711 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/5de868ed4eb24458868ece58321e576a is 50, key is test_row_0/A:col10/1731494302962/Put/seqid=0 2024-11-13T10:38:23,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741858_1034 (size=12241) 2024-11-13T10:38:23,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e0a2be93b1ea443185e78104c563b2f1 is 50, key is test_row_0/A:col10/1731494303673/Put/seqid=0 2024-11-13T10:38:23,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741859_1035 (size=12241) 2024-11-13T10:38:23,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741860_1036 (size=14341) 2024-11-13T10:38:23,810 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0170adf13ed6427e9a3eb0470e87f334 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0170adf13ed6427e9a3eb0470e87f334 2024-11-13T10:38:23,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e0a2be93b1ea443185e78104c563b2f1 2024-11-13T10:38:23,835 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/5de868ed4eb24458868ece58321e576a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5de868ed4eb24458868ece58321e576a 2024-11-13T10:38:23,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:23,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:23,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:23,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/35ba31e5a37b4f2e84eefd56cec49c27 is 50, key is test_row_0/B:col10/1731494303673/Put/seqid=0 2024-11-13T10:38:23,888 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into 0170adf13ed6427e9a3eb0470e87f334(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:23,888 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:23,888 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=13, startTime=1731494303664; duration=0sec 2024-11-13T10:38:23,888 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:23,888 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:23,888 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:23,890 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into 5de868ed4eb24458868ece58321e576a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:23,890 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:23,890 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=13, startTime=1731494303664; duration=0sec 2024-11-13T10:38:23,890 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:23,890 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:23,897 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:23,897 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:23,898 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:23,898 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e04feef815244765a6dd58b7ffdd7cf0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/32aef205c25143b9926796ba31505239, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/6d0acd039f614939ab650a651f8353da] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=35.3 K 2024-11-13T10:38:23,909 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e04feef815244765a6dd58b7ffdd7cf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731494301384 2024-11-13T10:38:23,911 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 32aef205c25143b9926796ba31505239, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731494301736 2024-11-13T10:38:23,912 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0acd039f614939ab650a651f8353da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731494302281 2024-11-13T10:38:23,968 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:23,969 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/81ede5f1d99b41689fb445948b8b245b is 50, key is test_row_0/C:col10/1731494302962/Put/seqid=0 2024-11-13T10:38:23,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741861_1037 (size=12001) 2024-11-13T10:38:23,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/35ba31e5a37b4f2e84eefd56cec49c27 2024-11-13T10:38:24,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/ffe40653a67b45e1935137f3a2da93d4 is 50, key is test_row_0/C:col10/1731494303673/Put/seqid=0 2024-11-13T10:38:24,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:24,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:24,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741862_1038 (size=12241) 2024-11-13T10:38:24,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T10:38:24,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741863_1039 (size=12001) 2024-11-13T10:38:24,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/ffe40653a67b45e1935137f3a2da93d4 2024-11-13T10:38:24,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e0a2be93b1ea443185e78104c563b2f1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0a2be93b1ea443185e78104c563b2f1 2024-11-13T10:38:24,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0a2be93b1ea443185e78104c563b2f1, entries=200, sequenceid=127, filesize=14.0 K 2024-11-13T10:38:24,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/35ba31e5a37b4f2e84eefd56cec49c27 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/35ba31e5a37b4f2e84eefd56cec49c27 2024-11-13T10:38:24,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/35ba31e5a37b4f2e84eefd56cec49c27, entries=150, sequenceid=127, filesize=11.7 K 2024-11-13T10:38:24,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/ffe40653a67b45e1935137f3a2da93d4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ffe40653a67b45e1935137f3a2da93d4 2024-11-13T10:38:24,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ffe40653a67b45e1935137f3a2da93d4, entries=150, sequenceid=127, filesize=11.7 K 2024-11-13T10:38:24,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=127.47 KB/130530 for 0704f3508555649d4fe7145badb2caf9 in 487ms, sequenceid=127, compaction requested=false 2024-11-13T10:38:24,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:24,182 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:24,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T10:38:24,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:24,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:24,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/fa3c1a217d714bcbb3c58fa4c2522122 is 50, key is test_row_0/A:col10/1731494304130/Put/seqid=0 2024-11-13T10:38:24,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741864_1040 (size=14541) 2024-11-13T10:38:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494364245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/fa3c1a217d714bcbb3c58fa4c2522122 2024-11-13T10:38:24,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494364250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494364249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494364252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494364264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/4421481e65a6447e9ee1678c455a9c6f is 50, key is test_row_0/B:col10/1731494304130/Put/seqid=0 2024-11-13T10:38:24,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741865_1041 (size=12151) 2024-11-13T10:38:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/4421481e65a6447e9ee1678c455a9c6f 2024-11-13T10:38:24,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494364377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/7aa5710b23754ed89bbd8d78bf726961 is 50, key is test_row_0/C:col10/1731494304130/Put/seqid=0 2024-11-13T10:38:24,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494364380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494364387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494364388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494364393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,448 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/81ede5f1d99b41689fb445948b8b245b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/81ede5f1d99b41689fb445948b8b245b 2024-11-13T10:38:24,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741866_1042 (size=12151) 2024-11-13T10:38:24,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/7aa5710b23754ed89bbd8d78bf726961 2024-11-13T10:38:24,463 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into 81ede5f1d99b41689fb445948b8b245b(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:24,463 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:24,463 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=13, startTime=1731494303664; duration=0sec 2024-11-13T10:38:24,464 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:24,464 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:24,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/fa3c1a217d714bcbb3c58fa4c2522122 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/fa3c1a217d714bcbb3c58fa4c2522122 2024-11-13T10:38:24,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/fa3c1a217d714bcbb3c58fa4c2522122, entries=200, sequenceid=152, filesize=14.2 K 2024-11-13T10:38:24,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/4421481e65a6447e9ee1678c455a9c6f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4421481e65a6447e9ee1678c455a9c6f 2024-11-13T10:38:24,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4421481e65a6447e9ee1678c455a9c6f, entries=150, sequenceid=152, filesize=11.9 K 2024-11-13T10:38:24,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/7aa5710b23754ed89bbd8d78bf726961 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7aa5710b23754ed89bbd8d78bf726961 2024-11-13T10:38:24,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:24,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:24,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7aa5710b23754ed89bbd8d78bf726961, entries=150, sequenceid=152, filesize=11.9 K 2024-11-13T10:38:24,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 0704f3508555649d4fe7145badb2caf9 in 342ms, sequenceid=152, compaction requested=true 2024-11-13T10:38:24,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:24,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:24,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:24,524 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:24,525 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:24,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:24,527 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41123 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:24,527 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:24,527 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,528 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5de868ed4eb24458868ece58321e576a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0a2be93b1ea443185e78104c563b2f1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/fa3c1a217d714bcbb3c58fa4c2522122] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=40.2 K 2024-11-13T10:38:24,528 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:24,528 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:24,528 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,528 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0170adf13ed6427e9a3eb0470e87f334, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/35ba31e5a37b4f2e84eefd56cec49c27, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4421481e65a6447e9ee1678c455a9c6f] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=35.5 K 2024-11-13T10:38:24,529 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0170adf13ed6427e9a3eb0470e87f334, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731494302281 2024-11-13T10:38:24,529 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de868ed4eb24458868ece58321e576a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731494302281 2024-11-13T10:38:24,530 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35ba31e5a37b4f2e84eefd56cec49c27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731494302989 2024-11-13T10:38:24,530 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e0a2be93b1ea443185e78104c563b2f1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731494302976 2024-11-13T10:38:24,531 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting fa3c1a217d714bcbb3c58fa4c2522122, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731494304074 2024-11-13T10:38:24,531 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4421481e65a6447e9ee1678c455a9c6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731494304122 2024-11-13T10:38:24,550 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:24,555 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/6cdce903582f4440b743aa30afe5668d is 50, key is test_row_0/A:col10/1731494304130/Put/seqid=0 2024-11-13T10:38:24,559 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:24,560 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/81882e752dec49989f7c31f0b0f3f0f2 is 50, key is test_row_0/B:col10/1731494304130/Put/seqid=0 2024-11-13T10:38:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:24,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T10:38:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741867_1043 (size=12493) 2024-11-13T10:38:24,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741868_1044 (size=12493) 2024-11-13T10:38:24,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/739e838c37c24d55b2606986f81026dd is 50, key is test_row_0/A:col10/1731494304248/Put/seqid=0 2024-11-13T10:38:24,650 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/81882e752dec49989f7c31f0b0f3f0f2 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/81882e752dec49989f7c31f0b0f3f0f2 2024-11-13T10:38:24,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741869_1045 (size=14541) 2024-11-13T10:38:24,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/739e838c37c24d55b2606986f81026dd 2024-11-13T10:38:24,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:24,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:24,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,677 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into 81882e752dec49989f7c31f0b0f3f0f2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:24,677 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:24,677 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=13, startTime=1731494304524; duration=0sec 2024-11-13T10:38:24,677 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:24,677 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:24,677 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,689 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:24,690 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:24,690 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,691 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/81ede5f1d99b41689fb445948b8b245b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ffe40653a67b45e1935137f3a2da93d4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7aa5710b23754ed89bbd8d78bf726961] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=35.5 K 2024-11-13T10:38:24,692 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ede5f1d99b41689fb445948b8b245b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731494302281 2024-11-13T10:38:24,693 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffe40653a67b45e1935137f3a2da93d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731494302989 2024-11-13T10:38:24,693 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aa5710b23754ed89bbd8d78bf726961, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731494304122 2024-11-13T10:38:24,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494364687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0939772bb412413a99c62e17774d4177 is 50, key is test_row_0/B:col10/1731494304248/Put/seqid=0 2024-11-13T10:38:24,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494364691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494364691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494364693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494364697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741870_1046 (size=12151) 2024-11-13T10:38:24,784 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:24,785 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/aa1ea1cb724d498e88b3d7baca314ad7 is 50, key is test_row_0/C:col10/1731494304130/Put/seqid=0 2024-11-13T10:38:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741871_1047 (size=12493) 2024-11-13T10:38:24,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494364839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494364839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494364839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494364839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494364842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:24,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:24,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:24,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,016 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/6cdce903582f4440b743aa30afe5668d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/6cdce903582f4440b743aa30afe5668d 2024-11-13T10:38:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,032 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into 6cdce903582f4440b743aa30afe5668d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:25,032 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:25,032 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=13, startTime=1731494304524; duration=0sec 2024-11-13T10:38:25,032 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:25,032 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:25,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494365047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494365048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494365048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T10:38:25,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494365058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494365059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:25,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:25,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0939772bb412413a99c62e17774d4177 2024-11-13T10:38:25,236 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/aa1ea1cb724d498e88b3d7baca314ad7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/aa1ea1cb724d498e88b3d7baca314ad7 2024-11-13T10:38:25,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/9ded69e2b1454ac09fb353601acdec1e is 50, key is test_row_0/C:col10/1731494304248/Put/seqid=0 2024-11-13T10:38:25,259 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into aa1ea1cb724d498e88b3d7baca314ad7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:25,259 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:25,259 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=13, startTime=1731494304525; duration=0sec 2024-11-13T10:38:25,260 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:25,260 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:25,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741872_1048 (size=12151) 2024-11-13T10:38:25,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/9ded69e2b1454ac09fb353601acdec1e 2024-11-13T10:38:25,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/739e838c37c24d55b2606986f81026dd as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/739e838c37c24d55b2606986f81026dd 2024-11-13T10:38:25,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/739e838c37c24d55b2606986f81026dd, entries=200, sequenceid=168, filesize=14.2 K 2024-11-13T10:38:25,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0939772bb412413a99c62e17774d4177 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0939772bb412413a99c62e17774d4177 2024-11-13T10:38:25,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:25,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0939772bb412413a99c62e17774d4177, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T10:38:25,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/9ded69e2b1454ac09fb353601acdec1e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9ded69e2b1454ac09fb353601acdec1e 2024-11-13T10:38:25,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9ded69e2b1454ac09fb353601acdec1e, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T10:38:25,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0704f3508555649d4fe7145badb2caf9 in 756ms, sequenceid=168, compaction requested=false 2024-11-13T10:38:25,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:25,366 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:25,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T10:38:25,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/d3b4ad150f794cbf808a1eafd7ee2dd2 is 50, key is test_row_0/A:col10/1731494305363/Put/seqid=0 2024-11-13T10:38:25,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741873_1049 (size=14541) 2024-11-13T10:38:25,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/d3b4ad150f794cbf808a1eafd7ee2dd2 2024-11-13T10:38:25,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494365402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494365403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494365405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494365407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494365406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/a4818fa54c7e41309342266923713e30 is 50, key is test_row_0/B:col10/1731494305363/Put/seqid=0 2024-11-13T10:38:25,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741874_1050 (size=12151) 2024-11-13T10:38:25,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:25,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:25,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494365517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494365518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494365528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494365527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494365529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:25,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:25,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494365727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494365739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494365739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494365740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494365741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:25,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:25,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:25,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/a4818fa54c7e41309342266923713e30 2024-11-13T10:38:25,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/3e4cdfdf88534984a3649ac95309bf0e is 50, key is test_row_0/C:col10/1731494305363/Put/seqid=0 2024-11-13T10:38:25,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741875_1051 (size=12151) 2024-11-13T10:38:25,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/3e4cdfdf88534984a3649ac95309bf0e 2024-11-13T10:38:25,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:25,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:25,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:25,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/d3b4ad150f794cbf808a1eafd7ee2dd2 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d3b4ad150f794cbf808a1eafd7ee2dd2 2024-11-13T10:38:25,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d3b4ad150f794cbf808a1eafd7ee2dd2, entries=200, sequenceid=193, filesize=14.2 K 2024-11-13T10:38:25,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/a4818fa54c7e41309342266923713e30 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/a4818fa54c7e41309342266923713e30 2024-11-13T10:38:26,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/a4818fa54c7e41309342266923713e30, entries=150, sequenceid=193, filesize=11.9 K 2024-11-13T10:38:26,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/3e4cdfdf88534984a3649ac95309bf0e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/3e4cdfdf88534984a3649ac95309bf0e 2024-11-13T10:38:26,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494366047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494366053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494366057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494366058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494366060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/3e4cdfdf88534984a3649ac95309bf0e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-13T10:38:26,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for 0704f3508555649d4fe7145badb2caf9 in 699ms, sequenceid=193, compaction requested=true 2024-11-13T10:38:26,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:26,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:26,065 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:26,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:26,066 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:26,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:26,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:26,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:26,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:26,068 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:26,068 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:26,068 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:26,068 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:26,068 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:26,068 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:26,068 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/6cdce903582f4440b743aa30afe5668d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/739e838c37c24d55b2606986f81026dd, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d3b4ad150f794cbf808a1eafd7ee2dd2] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=40.6 K 2024-11-13T10:38:26,069 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/81882e752dec49989f7c31f0b0f3f0f2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0939772bb412413a99c62e17774d4177, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/a4818fa54c7e41309342266923713e30] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=35.9 K 2024-11-13T10:38:26,069 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cdce903582f4440b743aa30afe5668d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731494304122 2024-11-13T10:38:26,070 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 739e838c37c24d55b2606986f81026dd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731494304248 2024-11-13T10:38:26,070 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 81882e752dec49989f7c31f0b0f3f0f2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731494304122 2024-11-13T10:38:26,071 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b4ad150f794cbf808a1eafd7ee2dd2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731494304676 2024-11-13T10:38:26,071 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0939772bb412413a99c62e17774d4177, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731494304248 2024-11-13T10:38:26,071 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting a4818fa54c7e41309342266923713e30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731494304676 2024-11-13T10:38:26,091 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:26,092 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0469d336fb0f4f1db8743c61edd72888 is 50, key is test_row_0/B:col10/1731494305363/Put/seqid=0 2024-11-13T10:38:26,103 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:26,103 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/de18b68979cb4867b2c3622cc84c77a0 is 50, key is test_row_0/A:col10/1731494305363/Put/seqid=0 2024-11-13T10:38:26,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T10:38:26,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:26,120 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T10:38:26,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741876_1052 (size=12595) 2024-11-13T10:38:26,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/5377abc56f744d8eae001894d6fc4ed8 is 50, key is test_row_0/A:col10/1731494305403/Put/seqid=0 2024-11-13T10:38:26,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741877_1053 (size=12595) 2024-11-13T10:38:26,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741878_1054 (size=12151) 2024-11-13T10:38:26,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:26,594 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0469d336fb0f4f1db8743c61edd72888 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0469d336fb0f4f1db8743c61edd72888 2024-11-13T10:38:26,598 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/5377abc56f744d8eae001894d6fc4ed8 2024-11-13T10:38:26,611 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/de18b68979cb4867b2c3622cc84c77a0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/de18b68979cb4867b2c3622cc84c77a0 2024-11-13T10:38:26,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,624 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into 0469d336fb0f4f1db8743c61edd72888(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494366613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,625 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:26,625 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=13, startTime=1731494306066; duration=0sec 2024-11-13T10:38:26,625 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:26,625 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into de18b68979cb4867b2c3622cc84c77a0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:26,625 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:26,625 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:26,625 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=13, startTime=1731494306065; duration=0sec 2024-11-13T10:38:26,625 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:26,625 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:26,625 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:26,628 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:26,628 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:26,628 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:26,628 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/aa1ea1cb724d498e88b3d7baca314ad7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9ded69e2b1454ac09fb353601acdec1e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/3e4cdfdf88534984a3649ac95309bf0e] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=35.9 K 2024-11-13T10:38:26,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/c037ee151db04ebaa84a6f71eb2bf589 is 50, key is test_row_0/B:col10/1731494305403/Put/seqid=0 2024-11-13T10:38:26,631 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting aa1ea1cb724d498e88b3d7baca314ad7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731494304122 2024-11-13T10:38:26,632 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ded69e2b1454ac09fb353601acdec1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731494304248 2024-11-13T10:38:26,632 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e4cdfdf88534984a3649ac95309bf0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731494304676 2024-11-13T10:38:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494366616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494366618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494366621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494366624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,655 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:26,656 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/ceb25a9a967649c38469251757fc9dbd is 50, key is test_row_0/C:col10/1731494305363/Put/seqid=0 2024-11-13T10:38:26,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741879_1055 (size=12151) 2024-11-13T10:38:26,693 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/c037ee151db04ebaa84a6f71eb2bf589 2024-11-13T10:38:26,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741880_1056 (size=12595) 2024-11-13T10:38:26,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/7f5eda8d0a76453da5150ddb099e579d is 50, key is test_row_0/C:col10/1731494305403/Put/seqid=0 2024-11-13T10:38:26,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494366739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494366748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494366757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494366758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494366757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741881_1057 (size=12151) 2024-11-13T10:38:26,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494366957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494366968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494366977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494366977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:26,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494366987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T10:38:27,128 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/ceb25a9a967649c38469251757fc9dbd as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ceb25a9a967649c38469251757fc9dbd 2024-11-13T10:38:27,151 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into ceb25a9a967649c38469251757fc9dbd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:27,151 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:27,151 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=13, startTime=1731494306066; duration=0sec 2024-11-13T10:38:27,151 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:27,151 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:27,209 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/7f5eda8d0a76453da5150ddb099e579d 2024-11-13T10:38:27,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/5377abc56f744d8eae001894d6fc4ed8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5377abc56f744d8eae001894d6fc4ed8 2024-11-13T10:38:27,230 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5377abc56f744d8eae001894d6fc4ed8, entries=150, sequenceid=209, filesize=11.9 K 2024-11-13T10:38:27,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/c037ee151db04ebaa84a6f71eb2bf589 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/c037ee151db04ebaa84a6f71eb2bf589 2024-11-13T10:38:27,249 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/c037ee151db04ebaa84a6f71eb2bf589, entries=150, sequenceid=209, filesize=11.9 K 2024-11-13T10:38:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/7f5eda8d0a76453da5150ddb099e579d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7f5eda8d0a76453da5150ddb099e579d 2024-11-13T10:38:27,257 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7f5eda8d0a76453da5150ddb099e579d, entries=150, sequenceid=209, filesize=11.9 K 2024-11-13T10:38:27,258 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 0704f3508555649d4fe7145badb2caf9 in 1138ms, sequenceid=209, compaction requested=false 2024-11-13T10:38:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T10:38:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T10:38:27,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T10:38:27,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3510 sec 2024-11-13T10:38:27,274 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:27,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T10:38:27,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 4.3620 sec 2024-11-13T10:38:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:27,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/bde13133d3de462bb8833f3df592695d is 50, key is test_row_0/A:col10/1731494306603/Put/seqid=0 2024-11-13T10:38:27,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741882_1058 (size=12151) 2024-11-13T10:38:27,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494367324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494367329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494367342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494367344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494367344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494367447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494367447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494367462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494367462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494367462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494367657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494367657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494367666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494367677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494367677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:27,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:27,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/bde13133d3de462bb8833f3df592695d 2024-11-13T10:38:27,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/9a7a040a330b47579d53f3132278d615 is 50, key is test_row_0/B:col10/1731494306603/Put/seqid=0 2024-11-13T10:38:27,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741883_1059 (size=12151) 2024-11-13T10:38:27,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/9a7a040a330b47579d53f3132278d615 2024-11-13T10:38:27,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/49308d357399458e8ba268c445f2a26d is 50, key is test_row_0/C:col10/1731494306603/Put/seqid=0 2024-11-13T10:38:27,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741884_1060 (size=12151) 2024-11-13T10:38:27,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/49308d357399458e8ba268c445f2a26d 2024-11-13T10:38:27,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/bde13133d3de462bb8833f3df592695d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/bde13133d3de462bb8833f3df592695d 2024-11-13T10:38:27,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/bde13133d3de462bb8833f3df592695d, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T10:38:27,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/9a7a040a330b47579d53f3132278d615 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/9a7a040a330b47579d53f3132278d615 2024-11-13T10:38:27,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/9a7a040a330b47579d53f3132278d615, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T10:38:27,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/49308d357399458e8ba268c445f2a26d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/49308d357399458e8ba268c445f2a26d 2024-11-13T10:38:27,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/49308d357399458e8ba268c445f2a26d, entries=150, sequenceid=233, filesize=11.9 K 2024-11-13T10:38:27,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 0704f3508555649d4fe7145badb2caf9 in 558ms, sequenceid=233, compaction requested=true 2024-11-13T10:38:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:27,833 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:27,833 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:27,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:27,834 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:27,834 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:27,834 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:27,835 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/de18b68979cb4867b2c3622cc84c77a0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5377abc56f744d8eae001894d6fc4ed8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/bde13133d3de462bb8833f3df592695d] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=36.0 K 2024-11-13T10:38:27,836 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:27,836 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting de18b68979cb4867b2c3622cc84c77a0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731494304676 2024-11-13T10:38:27,836 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:27,836 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:27,837 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0469d336fb0f4f1db8743c61edd72888, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/c037ee151db04ebaa84a6f71eb2bf589, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/9a7a040a330b47579d53f3132278d615] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=36.0 K 2024-11-13T10:38:27,837 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5377abc56f744d8eae001894d6fc4ed8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731494305394 2024-11-13T10:38:27,837 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0469d336fb0f4f1db8743c61edd72888, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731494304676 2024-11-13T10:38:27,837 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bde13133d3de462bb8833f3df592695d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731494306603 2024-11-13T10:38:27,838 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting c037ee151db04ebaa84a6f71eb2bf589, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731494305394 2024-11-13T10:38:27,839 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a7a040a330b47579d53f3132278d615, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731494306603 2024-11-13T10:38:27,865 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:27,866 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/eb1b5f2dde104f82a205900394fbef44 is 50, key is test_row_0/A:col10/1731494306603/Put/seqid=0 2024-11-13T10:38:27,873 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#49 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:27,874 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/351bbf6e178146cdb024f51bbb3ec851 is 50, key is test_row_0/B:col10/1731494306603/Put/seqid=0 2024-11-13T10:38:27,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741885_1061 (size=12697) 2024-11-13T10:38:27,903 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/eb1b5f2dde104f82a205900394fbef44 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/eb1b5f2dde104f82a205900394fbef44 2024-11-13T10:38:27,921 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into eb1b5f2dde104f82a205900394fbef44(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:27,921 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:27,921 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=13, startTime=1731494307833; duration=0sec 2024-11-13T10:38:27,926 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:27,926 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:27,926 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:27,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741886_1062 (size=12697) 2024-11-13T10:38:27,932 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:27,933 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:27,933 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:27,933 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ceb25a9a967649c38469251757fc9dbd, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7f5eda8d0a76453da5150ddb099e579d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/49308d357399458e8ba268c445f2a26d] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=36.0 K 2024-11-13T10:38:27,934 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceb25a9a967649c38469251757fc9dbd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731494304676 2024-11-13T10:38:27,948 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f5eda8d0a76453da5150ddb099e579d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731494305394 2024-11-13T10:38:27,949 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49308d357399458e8ba268c445f2a26d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731494306603 2024-11-13T10:38:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:27,979 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:27,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T10:38:27,996 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#50 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:27,997 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/831d6ae0bb20465f8bf00456f2e24100 is 50, key is test_row_0/C:col10/1731494306603/Put/seqid=0 2024-11-13T10:38:28,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/4eea08445c2e4a0d97fe453668ff88b0 is 50, key is test_row_0/A:col10/1731494307341/Put/seqid=0 2024-11-13T10:38:28,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494368034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494368036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494368039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741888_1064 (size=14541) 2024-11-13T10:38:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/4eea08445c2e4a0d97fe453668ff88b0 2024-11-13T10:38:28,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494368041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741887_1063 (size=12697) 2024-11-13T10:38:28,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494368056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,084 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/831d6ae0bb20465f8bf00456f2e24100 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/831d6ae0bb20465f8bf00456f2e24100 2024-11-13T10:38:28,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/7d1014cd1eb54edcbf44fb022ea0ea7d is 50, key is test_row_0/B:col10/1731494307341/Put/seqid=0 2024-11-13T10:38:28,104 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into 831d6ae0bb20465f8bf00456f2e24100(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:28,104 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:28,104 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=13, startTime=1731494307833; duration=0sec 2024-11-13T10:38:28,104 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:28,104 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:28,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741889_1065 (size=12151) 2024-11-13T10:38:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494368157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494368157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494368157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494368172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494368172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,356 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/351bbf6e178146cdb024f51bbb3ec851 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/351bbf6e178146cdb024f51bbb3ec851 2024-11-13T10:38:28,368 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into 351bbf6e178146cdb024f51bbb3ec851(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:28,368 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:28,368 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=13, startTime=1731494307833; duration=0sec 2024-11-13T10:38:28,369 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:28,369 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:28,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494368378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494368382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494368388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494368389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494368392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/7d1014cd1eb54edcbf44fb022ea0ea7d 2024-11-13T10:38:28,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/12d1911dd6654490a43c98323ad56edb is 50, key is test_row_0/C:col10/1731494307341/Put/seqid=0 2024-11-13T10:38:28,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741890_1066 (size=12151) 2024-11-13T10:38:28,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494368687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494368701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494368705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494368706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494368708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:28,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/12d1911dd6654490a43c98323ad56edb 2024-11-13T10:38:29,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/4eea08445c2e4a0d97fe453668ff88b0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4eea08445c2e4a0d97fe453668ff88b0 2024-11-13T10:38:29,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4eea08445c2e4a0d97fe453668ff88b0, entries=200, sequenceid=250, filesize=14.2 K 2024-11-13T10:38:29,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/7d1014cd1eb54edcbf44fb022ea0ea7d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/7d1014cd1eb54edcbf44fb022ea0ea7d 2024-11-13T10:38:29,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/7d1014cd1eb54edcbf44fb022ea0ea7d, entries=150, sequenceid=250, filesize=11.9 K 2024-11-13T10:38:29,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/12d1911dd6654490a43c98323ad56edb as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/12d1911dd6654490a43c98323ad56edb 2024-11-13T10:38:29,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/12d1911dd6654490a43c98323ad56edb, entries=150, sequenceid=250, filesize=11.9 K 2024-11-13T10:38:29,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 0704f3508555649d4fe7145badb2caf9 in 1158ms, sequenceid=250, compaction requested=false 2024-11-13T10:38:29,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:29,215 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:29,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T10:38:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:29,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/24e2ec08f9b94d8dbc190f7034d01235 is 50, key is test_row_0/A:col10/1731494308029/Put/seqid=0 2024-11-13T10:38:29,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741891_1067 (size=14741) 2024-11-13T10:38:29,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/24e2ec08f9b94d8dbc190f7034d01235 2024-11-13T10:38:29,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/fe9aaf3c53944ea990e3cd38c02a7fb3 is 50, key is test_row_0/B:col10/1731494308029/Put/seqid=0 2024-11-13T10:38:29,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741892_1068 (size=12301) 2024-11-13T10:38:29,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494369394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494369402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494369409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494369411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494369411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494369517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494369527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494369527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494369528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494369528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/fe9aaf3c53944ea990e3cd38c02a7fb3 2024-11-13T10:38:29,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/e9b3d5e4f7184b298b0fb3ffa4efda84 is 50, key is test_row_0/C:col10/1731494308029/Put/seqid=0 2024-11-13T10:38:29,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494369737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494369737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494369738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494369738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494369738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:29,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:29,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:29,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741893_1069 (size=12301) 2024-11-13T10:38:29,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/e9b3d5e4f7184b298b0fb3ffa4efda84 2024-11-13T10:38:29,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/24e2ec08f9b94d8dbc190f7034d01235 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/24e2ec08f9b94d8dbc190f7034d01235 2024-11-13T10:38:29,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/24e2ec08f9b94d8dbc190f7034d01235, entries=200, sequenceid=273, filesize=14.4 K 2024-11-13T10:38:29,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/fe9aaf3c53944ea990e3cd38c02a7fb3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fe9aaf3c53944ea990e3cd38c02a7fb3 2024-11-13T10:38:29,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fe9aaf3c53944ea990e3cd38c02a7fb3, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T10:38:29,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/e9b3d5e4f7184b298b0fb3ffa4efda84 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e9b3d5e4f7184b298b0fb3ffa4efda84 2024-11-13T10:38:29,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e9b3d5e4f7184b298b0fb3ffa4efda84, entries=150, sequenceid=273, filesize=12.0 K 2024-11-13T10:38:29,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 0704f3508555649d4fe7145badb2caf9 in 590ms, sequenceid=273, compaction requested=true 2024-11-13T10:38:29,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:29,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:29,806 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:29,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:29,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:29,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:29,806 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:29,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:29,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:29,807 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:29,807 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:29,807 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:29,808 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:29,808 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:29,808 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/eb1b5f2dde104f82a205900394fbef44, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4eea08445c2e4a0d97fe453668ff88b0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/24e2ec08f9b94d8dbc190f7034d01235] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=41.0 K 2024-11-13T10:38:29,808 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:29,808 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/351bbf6e178146cdb024f51bbb3ec851, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/7d1014cd1eb54edcbf44fb022ea0ea7d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fe9aaf3c53944ea990e3cd38c02a7fb3] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=36.3 K 2024-11-13T10:38:29,817 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 351bbf6e178146cdb024f51bbb3ec851, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731494306603 2024-11-13T10:38:29,817 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb1b5f2dde104f82a205900394fbef44, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731494306603 2024-11-13T10:38:29,818 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d1014cd1eb54edcbf44fb022ea0ea7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494307336 2024-11-13T10:38:29,818 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eea08445c2e4a0d97fe453668ff88b0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494307308 2024-11-13T10:38:29,818 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9aaf3c53944ea990e3cd38c02a7fb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731494308029 2024-11-13T10:38:29,819 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e2ec08f9b94d8dbc190f7034d01235, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731494308029 2024-11-13T10:38:29,847 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:29,848 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/17eb824552cc45169b00a0fa710f7257 is 50, key is test_row_0/A:col10/1731494308029/Put/seqid=0 2024-11-13T10:38:29,859 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:29,860 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0375ea0dee18406392fa9646bcd214f7 is 50, key is test_row_0/B:col10/1731494308029/Put/seqid=0 2024-11-13T10:38:29,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741895_1071 (size=12949) 2024-11-13T10:38:29,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741894_1070 (size=12949) 2024-11-13T10:38:29,922 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0375ea0dee18406392fa9646bcd214f7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0375ea0dee18406392fa9646bcd214f7 2024-11-13T10:38:29,938 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into 0375ea0dee18406392fa9646bcd214f7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:29,938 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:29,938 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=13, startTime=1731494309806; duration=0sec 2024-11-13T10:38:29,938 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:29,938 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:29,941 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:29,942 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/17eb824552cc45169b00a0fa710f7257 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/17eb824552cc45169b00a0fa710f7257 2024-11-13T10:38:29,943 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:29,943 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:29,944 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:29,944 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/831d6ae0bb20465f8bf00456f2e24100, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/12d1911dd6654490a43c98323ad56edb, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e9b3d5e4f7184b298b0fb3ffa4efda84] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=36.3 K 2024-11-13T10:38:29,944 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 831d6ae0bb20465f8bf00456f2e24100, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731494306603 2024-11-13T10:38:29,945 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 12d1911dd6654490a43c98323ad56edb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494307336 2024-11-13T10:38:29,945 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b3d5e4f7184b298b0fb3ffa4efda84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731494308029 2024-11-13T10:38:29,952 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into 17eb824552cc45169b00a0fa710f7257(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:29,952 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:29,952 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=13, startTime=1731494309806; duration=0sec 2024-11-13T10:38:29,953 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:29,953 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:29,980 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:29,981 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/ea90f7fd37724b2e969681f5a4d7bacd is 50, key is test_row_0/C:col10/1731494308029/Put/seqid=0 2024-11-13T10:38:30,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741896_1072 (size=12949) 2024-11-13T10:38:30,024 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/ea90f7fd37724b2e969681f5a4d7bacd as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ea90f7fd37724b2e969681f5a4d7bacd 2024-11-13T10:38:30,044 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into ea90f7fd37724b2e969681f5a4d7bacd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:30,044 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:30,045 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=13, startTime=1731494309806; duration=0sec 2024-11-13T10:38:30,045 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:30,045 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:30,053 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:30,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T10:38:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:30,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/f2a5c99ac18b402197b067a4f18022e6 is 50, key is test_row_0/A:col10/1731494310052/Put/seqid=0 2024-11-13T10:38:30,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741897_1073 (size=17181) 2024-11-13T10:38:30,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494370146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494370147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494370150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494370163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494370161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494370267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494370267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494370268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494370277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494370287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494370478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494370478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494370478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/f2a5c99ac18b402197b067a4f18022e6 2024-11-13T10:38:30,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494370498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494370499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/af78768d2df04cb7965b8d92e2a96249 is 50, key is test_row_0/B:col10/1731494310052/Put/seqid=0 2024-11-13T10:38:30,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741898_1074 (size=12301) 2024-11-13T10:38:30,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/af78768d2df04cb7965b8d92e2a96249 2024-11-13T10:38:30,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c36b6b1677b2415fbccdef7b75eac90d is 50, key is test_row_0/C:col10/1731494310052/Put/seqid=0 2024-11-13T10:38:30,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741899_1075 (size=12301) 2024-11-13T10:38:30,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494370787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494370787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494370787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494370817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494370817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c36b6b1677b2415fbccdef7b75eac90d 2024-11-13T10:38:31,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/f2a5c99ac18b402197b067a4f18022e6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f2a5c99ac18b402197b067a4f18022e6 2024-11-13T10:38:31,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f2a5c99ac18b402197b067a4f18022e6, entries=250, sequenceid=294, filesize=16.8 K 2024-11-13T10:38:31,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/af78768d2df04cb7965b8d92e2a96249 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/af78768d2df04cb7965b8d92e2a96249 2024-11-13T10:38:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T10:38:31,097 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:31,098 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:31,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/af78768d2df04cb7965b8d92e2a96249, entries=150, sequenceid=294, filesize=12.0 K 2024-11-13T10:38:31,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c36b6b1677b2415fbccdef7b75eac90d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c36b6b1677b2415fbccdef7b75eac90d 2024-11-13T10:38:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-13T10:38:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T10:38:31,105 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:31,106 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:31,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:31,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c36b6b1677b2415fbccdef7b75eac90d, entries=150, sequenceid=294, filesize=12.0 K 2024-11-13T10:38:31,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 0704f3508555649d4fe7145badb2caf9 in 1067ms, sequenceid=294, compaction requested=false 2024-11-13T10:38:31,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T10:38:31,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T10:38:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:31,261 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T10:38:31,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/d051f0fe35a64dec9a80360d10e442ad is 50, key is test_row_0/A:col10/1731494310134/Put/seqid=0 2024-11-13T10:38:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:31,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:31,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741900_1076 (size=12301) 2024-11-13T10:38:31,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494371345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494371354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494371354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494371361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494371362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T10:38:31,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494371467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494371467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494371477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494371477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494371477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494371677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494371677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494371687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494371688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494371688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:31,720 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/d051f0fe35a64dec9a80360d10e442ad 2024-11-13T10:38:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T10:38:31,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/dab7983f9882433383fb8587ac9f629b is 50, key is test_row_0/B:col10/1731494310134/Put/seqid=0 2024-11-13T10:38:31,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741901_1077 (size=12301) 2024-11-13T10:38:31,752 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/dab7983f9882433383fb8587ac9f629b 2024-11-13T10:38:31,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/075b305a72bf49cf8d79e5e2a5644eae is 50, key is test_row_0/C:col10/1731494310134/Put/seqid=0 2024-11-13T10:38:31,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741902_1078 (size=12301) 2024-11-13T10:38:31,791 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/075b305a72bf49cf8d79e5e2a5644eae 2024-11-13T10:38:31,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/d051f0fe35a64dec9a80360d10e442ad as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d051f0fe35a64dec9a80360d10e442ad 2024-11-13T10:38:31,818 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d051f0fe35a64dec9a80360d10e442ad, entries=150, sequenceid=313, filesize=12.0 K 2024-11-13T10:38:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/dab7983f9882433383fb8587ac9f629b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/dab7983f9882433383fb8587ac9f629b 2024-11-13T10:38:31,833 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/dab7983f9882433383fb8587ac9f629b, entries=150, sequenceid=313, filesize=12.0 K 2024-11-13T10:38:31,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/075b305a72bf49cf8d79e5e2a5644eae as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/075b305a72bf49cf8d79e5e2a5644eae 2024-11-13T10:38:31,846 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/075b305a72bf49cf8d79e5e2a5644eae, entries=150, sequenceid=313, filesize=12.0 K 2024-11-13T10:38:31,848 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 0704f3508555649d4fe7145badb2caf9 in 587ms, sequenceid=313, compaction requested=true 2024-11-13T10:38:31,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:31,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:31,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-13T10:38:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-13T10:38:31,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-13T10:38:31,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 750 msec 2024-11-13T10:38:31,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 763 msec 2024-11-13T10:38:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:32,000 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:32,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T10:38:32,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/7258074b175d4d6da9c6d2df3cd31f39 is 50, key is test_row_0/A:col10/1731494311344/Put/seqid=0 2024-11-13T10:38:32,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741903_1079 (size=17181) 2024-11-13T10:38:32,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/7258074b175d4d6da9c6d2df3cd31f39 2024-11-13T10:38:32,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494372028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494372028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/899636249f304be09ab516feae8292a8 is 50, key is test_row_0/B:col10/1731494311344/Put/seqid=0 2024-11-13T10:38:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494372033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494372034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494372038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741904_1080 (size=12301) 2024-11-13T10:38:32,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494372136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494372137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494372161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494372164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494372164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T10:38:32,237 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:32,238 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-13T10:38:32,241 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:32,243 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:32,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T10:38:32,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494372349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T10:38:32,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494372352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494372376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494372377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494372384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:32,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:32,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/899636249f304be09ab516feae8292a8 2024-11-13T10:38:32,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/1235f26a6bbf46f5aa94b38c4a52d051 is 50, key is test_row_0/C:col10/1731494311344/Put/seqid=0 2024-11-13T10:38:32,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741905_1081 (size=12301) 2024-11-13T10:38:32,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/1235f26a6bbf46f5aa94b38c4a52d051 2024-11-13T10:38:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/7258074b175d4d6da9c6d2df3cd31f39 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7258074b175d4d6da9c6d2df3cd31f39 2024-11-13T10:38:32,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:32,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:32,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T10:38:32,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7258074b175d4d6da9c6d2df3cd31f39, entries=250, sequenceid=333, filesize=16.8 K 2024-11-13T10:38:32,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/899636249f304be09ab516feae8292a8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/899636249f304be09ab516feae8292a8 2024-11-13T10:38:32,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/899636249f304be09ab516feae8292a8, entries=150, sequenceid=333, filesize=12.0 K 2024-11-13T10:38:32,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/1235f26a6bbf46f5aa94b38c4a52d051 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/1235f26a6bbf46f5aa94b38c4a52d051 2024-11-13T10:38:32,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/1235f26a6bbf46f5aa94b38c4a52d051, entries=150, sequenceid=333, filesize=12.0 K 2024-11-13T10:38:32,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 0704f3508555649d4fe7145badb2caf9 in 630ms, sequenceid=333, compaction requested=true 2024-11-13T10:38:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:38:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T10:38:32,631 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:32,632 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:32,636 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:32,636 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:32,636 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,637 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/17eb824552cc45169b00a0fa710f7257, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f2a5c99ac18b402197b067a4f18022e6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d051f0fe35a64dec9a80360d10e442ad, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7258074b175d4d6da9c6d2df3cd31f39] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=58.2 K 2024-11-13T10:38:32,637 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 17eb824552cc45169b00a0fa710f7257, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731494308029 2024-11-13T10:38:32,640 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:32,641 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:32,641 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,641 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ea90f7fd37724b2e969681f5a4d7bacd, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c36b6b1677b2415fbccdef7b75eac90d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/075b305a72bf49cf8d79e5e2a5644eae, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/1235f26a6bbf46f5aa94b38c4a52d051] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=48.7 K 2024-11-13T10:38:32,641 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a5c99ac18b402197b067a4f18022e6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731494309390 2024-11-13T10:38:32,646 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea90f7fd37724b2e969681f5a4d7bacd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731494308029 2024-11-13T10:38:32,646 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting d051f0fe35a64dec9a80360d10e442ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731494310134 2024-11-13T10:38:32,646 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c36b6b1677b2415fbccdef7b75eac90d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731494309408 2024-11-13T10:38:32,647 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7258074b175d4d6da9c6d2df3cd31f39, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731494311344 2024-11-13T10:38:32,647 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 075b305a72bf49cf8d79e5e2a5644eae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731494310134 2024-11-13T10:38:32,648 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1235f26a6bbf46f5aa94b38c4a52d051, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731494311344 2024-11-13T10:38:32,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:32,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T10:38:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:32,686 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:32,687 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c84091d1e6fb4effa18bd7c7478a2828 is 50, key is test_row_0/C:col10/1731494311344/Put/seqid=0 2024-11-13T10:38:32,690 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:32,691 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/4155a5b633df4588b21c81f341459785 is 50, key is test_row_0/A:col10/1731494311344/Put/seqid=0 2024-11-13T10:38:32,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/c4657ca606644e58b58058b31035f834 is 50, key is test_row_0/A:col10/1731494312674/Put/seqid=0 2024-11-13T10:38:32,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741906_1082 (size=13085) 2024-11-13T10:38:32,729 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c84091d1e6fb4effa18bd7c7478a2828 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c84091d1e6fb4effa18bd7c7478a2828 2024-11-13T10:38:32,737 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into c84091d1e6fb4effa18bd7c7478a2828(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:32,737 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:32,737 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=12, startTime=1731494312631; duration=0sec 2024-11-13T10:38:32,737 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:32,737 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:32,737 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:32,739 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:32,739 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:32,739 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,739 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0375ea0dee18406392fa9646bcd214f7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/af78768d2df04cb7965b8d92e2a96249, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/dab7983f9882433383fb8587ac9f629b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/899636249f304be09ab516feae8292a8] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=48.7 K 2024-11-13T10:38:32,740 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0375ea0dee18406392fa9646bcd214f7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731494308029 2024-11-13T10:38:32,742 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting af78768d2df04cb7965b8d92e2a96249, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731494309408 2024-11-13T10:38:32,745 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting dab7983f9882433383fb8587ac9f629b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731494310134 2024-11-13T10:38:32,745 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 899636249f304be09ab516feae8292a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731494311344 2024-11-13T10:38:32,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494372729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494372730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494372739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494372742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494372746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741907_1083 (size=13085) 2024-11-13T10:38:32,768 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/4155a5b633df4588b21c81f341459785 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4155a5b633df4588b21c81f341459785 2024-11-13T10:38:32,778 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into 4155a5b633df4588b21c81f341459785(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:32,778 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:32,779 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=12, startTime=1731494312631; duration=0sec 2024-11-13T10:38:32,779 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:32,779 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:32,783 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:32,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741908_1084 (size=14741) 2024-11-13T10:38:32,784 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/d1692f4f79a44812aba1d8b0d14e5e09 is 50, key is test_row_0/B:col10/1731494311344/Put/seqid=0 2024-11-13T10:38:32,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/c4657ca606644e58b58058b31035f834 2024-11-13T10:38:32,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741909_1085 (size=13085) 2024-11-13T10:38:32,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/24e88cdf878a413b958de0d22c6e5bcc is 50, key is test_row_0/B:col10/1731494312674/Put/seqid=0 2024-11-13T10:38:32,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494372858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494372858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494372860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494372862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494372869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:32,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:32,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:32,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:32,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:32,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T10:38:32,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741910_1086 (size=12301) 2024-11-13T10:38:33,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:33,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:33,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494373066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494373077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494373077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494373077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494373077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,239 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/d1692f4f79a44812aba1d8b0d14e5e09 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/d1692f4f79a44812aba1d8b0d14e5e09 2024-11-13T10:38:33,256 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into d1692f4f79a44812aba1d8b0d14e5e09(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:33,256 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:33,256 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=12, startTime=1731494312631; duration=0sec 2024-11-13T10:38:33,256 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:33,257 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:33,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/24e88cdf878a413b958de0d22c6e5bcc 2024-11-13T10:38:33,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/e3947ec4f3ed44d4a9bf4af704596130 is 50, key is test_row_0/C:col10/1731494312674/Put/seqid=0 2024-11-13T10:38:33,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741911_1087 (size=12301) 2024-11-13T10:38:33,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/e3947ec4f3ed44d4a9bf4af704596130 2024-11-13T10:38:33,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/c4657ca606644e58b58058b31035f834 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/c4657ca606644e58b58058b31035f834 2024-11-13T10:38:33,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/c4657ca606644e58b58058b31035f834, entries=200, sequenceid=350, filesize=14.4 K 2024-11-13T10:38:33,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/24e88cdf878a413b958de0d22c6e5bcc as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/24e88cdf878a413b958de0d22c6e5bcc 2024-11-13T10:38:33,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:33,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:33,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/24e88cdf878a413b958de0d22c6e5bcc, entries=150, sequenceid=350, filesize=12.0 K 2024-11-13T10:38:33,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/e3947ec4f3ed44d4a9bf4af704596130 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e3947ec4f3ed44d4a9bf4af704596130 2024-11-13T10:38:33,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e3947ec4f3ed44d4a9bf4af704596130, entries=150, sequenceid=350, filesize=12.0 K 2024-11-13T10:38:33,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 0704f3508555649d4fe7145badb2caf9 in 693ms, sequenceid=350, compaction requested=false 2024-11-13T10:38:33,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:33,383 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:33,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T10:38:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T10:38:33,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/8d3651ad1fe4447899a55efe29c195f7 is 50, key is test_row_0/A:col10/1731494312716/Put/seqid=0 2024-11-13T10:38:33,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741912_1088 (size=12301) 2024-11-13T10:38:33,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/8d3651ad1fe4447899a55efe29c195f7 2024-11-13T10:38:33,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494373401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494373402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494373404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494373412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494373417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/cd23e79ce47346c4b38dbb3115670bf9 is 50, key is test_row_0/B:col10/1731494312716/Put/seqid=0 2024-11-13T10:38:33,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741913_1089 (size=12301) 2024-11-13T10:38:33,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:33,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:33,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494373518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494373519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494373519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494373527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494373527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494373727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494373727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494373736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494373737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494373739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:33,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:33,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:33,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:33,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/cd23e79ce47346c4b38dbb3115670bf9 2024-11-13T10:38:33,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/75c8d002a09344e7afc7df0578b18051 is 50, key is test_row_0/C:col10/1731494312716/Put/seqid=0 2024-11-13T10:38:33,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741914_1090 (size=12301) 2024-11-13T10:38:33,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/75c8d002a09344e7afc7df0578b18051 2024-11-13T10:38:33,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/8d3651ad1fe4447899a55efe29c195f7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8d3651ad1fe4447899a55efe29c195f7 2024-11-13T10:38:33,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8d3651ad1fe4447899a55efe29c195f7, entries=150, sequenceid=373, filesize=12.0 K 2024-11-13T10:38:33,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/cd23e79ce47346c4b38dbb3115670bf9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/cd23e79ce47346c4b38dbb3115670bf9 2024-11-13T10:38:33,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:33,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:33,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:33,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/cd23e79ce47346c4b38dbb3115670bf9, entries=150, sequenceid=373, filesize=12.0 K 2024-11-13T10:38:33,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/75c8d002a09344e7afc7df0578b18051 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/75c8d002a09344e7afc7df0578b18051 2024-11-13T10:38:33,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/75c8d002a09344e7afc7df0578b18051, entries=150, sequenceid=373, filesize=12.0 K 2024-11-13T10:38:33,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 0704f3508555649d4fe7145badb2caf9 in 606ms, sequenceid=373, compaction requested=true 2024-11-13T10:38:33,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:33,990 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:33,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:33,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:33,990 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:33,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:33,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:33,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:33,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:33,992 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:33,992 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:33,992 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,992 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4155a5b633df4588b21c81f341459785, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/c4657ca606644e58b58058b31035f834, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8d3651ad1fe4447899a55efe29c195f7] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=39.2 K 2024-11-13T10:38:33,993 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:33,993 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:33,993 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:33,993 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/d1692f4f79a44812aba1d8b0d14e5e09, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/24e88cdf878a413b958de0d22c6e5bcc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/cd23e79ce47346c4b38dbb3115670bf9] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=36.8 K 2024-11-13T10:38:33,993 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4155a5b633df4588b21c81f341459785, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731494311344 2024-11-13T10:38:33,994 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4657ca606644e58b58058b31035f834, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731494312025 2024-11-13T10:38:33,995 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d3651ad1fe4447899a55efe29c195f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731494312716 2024-11-13T10:38:34,002 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting d1692f4f79a44812aba1d8b0d14e5e09, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731494311344 2024-11-13T10:38:34,003 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e88cdf878a413b958de0d22c6e5bcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731494312031 2024-11-13T10:38:34,004 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting cd23e79ce47346c4b38dbb3115670bf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731494312716 2024-11-13T10:38:34,019 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:34,022 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/7f06161599ee41179493c4c1747f7826 is 50, key is test_row_0/A:col10/1731494312716/Put/seqid=0 2024-11-13T10:38:34,035 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:34,036 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/85e104e6f219421384aedd2e421cc8c2 is 50, key is test_row_0/B:col10/1731494312716/Put/seqid=0 2024-11-13T10:38:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:34,040 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:34,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T10:38:34,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e12850ca5a724ddc96a743e040c2ca8a is 50, key is test_row_0/A:col10/1731494313409/Put/seqid=0 2024-11-13T10:38:34,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741915_1091 (size=13187) 2024-11-13T10:38:34,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494374082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494374086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494374092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494374094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,114 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/7f06161599ee41179493c4c1747f7826 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7f06161599ee41179493c4c1747f7826 2024-11-13T10:38:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494374113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,130 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into 7f06161599ee41179493c4c1747f7826(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:34,130 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:34,130 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=13, startTime=1731494313990; duration=0sec 2024-11-13T10:38:34,130 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:34,130 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:34,130 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:34,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741917_1093 (size=14741) 2024-11-13T10:38:34,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e12850ca5a724ddc96a743e040c2ca8a 2024-11-13T10:38:34,139 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:34,139 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:34,140 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,140 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c84091d1e6fb4effa18bd7c7478a2828, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e3947ec4f3ed44d4a9bf4af704596130, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/75c8d002a09344e7afc7df0578b18051] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=36.8 K 2024-11-13T10:38:34,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,141 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c84091d1e6fb4effa18bd7c7478a2828, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731494311344 2024-11-13T10:38:34,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,141 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3947ec4f3ed44d4a9bf4af704596130, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731494312031 2024-11-13T10:38:34,143 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c8d002a09344e7afc7df0578b18051, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731494312716 2024-11-13T10:38:34,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/4e82630c85f54580a8831cc5f8e0f8e1 is 50, key is test_row_0/B:col10/1731494313409/Put/seqid=0 2024-11-13T10:38:34,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741916_1092 (size=13187) 2024-11-13T10:38:34,163 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:34,164 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/63f4bab7dbd948aeb47d7f3d62319545 is 50, key is test_row_0/C:col10/1731494312716/Put/seqid=0 2024-11-13T10:38:34,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494374208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494374208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494374210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494374227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494374227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741918_1094 (size=12301) 2024-11-13T10:38:34,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/4e82630c85f54580a8831cc5f8e0f8e1 2024-11-13T10:38:34,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741919_1095 (size=13187) 2024-11-13T10:38:34,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/8bf551882b034733a8090eca311029a9 is 50, key is test_row_0/C:col10/1731494313409/Put/seqid=0 2024-11-13T10:38:34,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741920_1096 (size=12301) 2024-11-13T10:38:34,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T10:38:34,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494374427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494374428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494374428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494374437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494374437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:34,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:34,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,577 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/85e104e6f219421384aedd2e421cc8c2 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/85e104e6f219421384aedd2e421cc8c2 2024-11-13T10:38:34,589 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into 85e104e6f219421384aedd2e421cc8c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:34,589 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:34,589 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=13, startTime=1731494313990; duration=0sec 2024-11-13T10:38:34,590 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:34,590 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:34,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:34,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,692 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/63f4bab7dbd948aeb47d7f3d62319545 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/63f4bab7dbd948aeb47d7f3d62319545 2024-11-13T10:38:34,703 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into 63f4bab7dbd948aeb47d7f3d62319545(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:34,703 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:34,703 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=13, startTime=1731494313991; duration=0sec 2024-11-13T10:38:34,703 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:34,703 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:34,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/8bf551882b034733a8090eca311029a9 2024-11-13T10:38:34,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/e12850ca5a724ddc96a743e040c2ca8a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e12850ca5a724ddc96a743e040c2ca8a 2024-11-13T10:38:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494374736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494374737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e12850ca5a724ddc96a743e040c2ca8a, entries=200, sequenceid=392, filesize=14.4 K 2024-11-13T10:38:34,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/4e82630c85f54580a8831cc5f8e0f8e1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e82630c85f54580a8831cc5f8e0f8e1 2024-11-13T10:38:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494374750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494374750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494374757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:34,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e82630c85f54580a8831cc5f8e0f8e1, entries=150, sequenceid=392, filesize=12.0 K 2024-11-13T10:38:34,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:34,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/8bf551882b034733a8090eca311029a9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/8bf551882b034733a8090eca311029a9 2024-11-13T10:38:34,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/8bf551882b034733a8090eca311029a9, entries=150, sequenceid=392, filesize=12.0 K 2024-11-13T10:38:34,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 0704f3508555649d4fe7145badb2caf9 in 755ms, sequenceid=392, compaction requested=false 2024-11-13T10:38:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:34,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T10:38:34,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:34,933 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T10:38:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/137d52b27690435a8173932121c8f4c6 is 50, key is test_row_0/A:col10/1731494314091/Put/seqid=0 2024-11-13T10:38:34,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741921_1097 (size=12301) 2024-11-13T10:38:34,971 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/137d52b27690435a8173932121c8f4c6 2024-11-13T10:38:34,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/222fa8e0ab7f45bd8a19df50488b18a7 is 50, key is test_row_0/B:col10/1731494314091/Put/seqid=0 2024-11-13T10:38:34,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741922_1098 (size=12301) 2024-11-13T10:38:35,006 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/222fa8e0ab7f45bd8a19df50488b18a7 2024-11-13T10:38:35,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/920e298aacc14fc5b4f9a251c52bda14 is 50, key is test_row_0/C:col10/1731494314091/Put/seqid=0 2024-11-13T10:38:35,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741923_1099 (size=12301) 2024-11-13T10:38:35,068 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/920e298aacc14fc5b4f9a251c52bda14 2024-11-13T10:38:35,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/137d52b27690435a8173932121c8f4c6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/137d52b27690435a8173932121c8f4c6 2024-11-13T10:38:35,095 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/137d52b27690435a8173932121c8f4c6, entries=150, sequenceid=413, filesize=12.0 K 2024-11-13T10:38:35,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/222fa8e0ab7f45bd8a19df50488b18a7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/222fa8e0ab7f45bd8a19df50488b18a7 2024-11-13T10:38:35,107 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/222fa8e0ab7f45bd8a19df50488b18a7, entries=150, sequenceid=413, filesize=12.0 K 2024-11-13T10:38:35,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/920e298aacc14fc5b4f9a251c52bda14 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/920e298aacc14fc5b4f9a251c52bda14 2024-11-13T10:38:35,126 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/920e298aacc14fc5b4f9a251c52bda14, entries=150, sequenceid=413, filesize=12.0 K 2024-11-13T10:38:35,129 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 0704f3508555649d4fe7145badb2caf9 in 195ms, sequenceid=413, compaction requested=true 2024-11-13T10:38:35,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:35,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:35,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-13T10:38:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-13T10:38:35,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T10:38:35,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8940 sec 2024-11-13T10:38:35,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.9060 sec 2024-11-13T10:38:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:35,260 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:35,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T10:38:35,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/48731492357447cf9aade763b81ba317 is 50, key is test_row_0/A:col10/1731494315256/Put/seqid=0 2024-11-13T10:38:35,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494375292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494375291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494375294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494375296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494375297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741924_1100 (size=17181) 2024-11-13T10:38:35,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/48731492357447cf9aade763b81ba317 2024-11-13T10:38:35,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/731f8f4b04484313a828f4f5fa5ae6c1 is 50, key is test_row_0/B:col10/1731494315256/Put/seqid=0 2024-11-13T10:38:35,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741925_1101 (size=12301) 2024-11-13T10:38:35,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/731f8f4b04484313a828f4f5fa5ae6c1 2024-11-13T10:38:35,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/0d2fdc93e02748948d4e1072f839b9e1 is 50, key is test_row_0/C:col10/1731494315256/Put/seqid=0 2024-11-13T10:38:35,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741926_1102 (size=12301) 2024-11-13T10:38:35,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494375407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494375407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494375407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494375407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494375407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494375617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494375627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494375627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494375627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494375628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/0d2fdc93e02748948d4e1072f839b9e1 2024-11-13T10:38:35,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/48731492357447cf9aade763b81ba317 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/48731492357447cf9aade763b81ba317 2024-11-13T10:38:35,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/48731492357447cf9aade763b81ba317, entries=250, sequenceid=425, filesize=16.8 K 2024-11-13T10:38:35,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/731f8f4b04484313a828f4f5fa5ae6c1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/731f8f4b04484313a828f4f5fa5ae6c1 2024-11-13T10:38:35,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/731f8f4b04484313a828f4f5fa5ae6c1, entries=150, sequenceid=425, filesize=12.0 K 2024-11-13T10:38:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/0d2fdc93e02748948d4e1072f839b9e1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0d2fdc93e02748948d4e1072f839b9e1 2024-11-13T10:38:35,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0d2fdc93e02748948d4e1072f839b9e1, entries=150, sequenceid=425, filesize=12.0 K 2024-11-13T10:38:35,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0704f3508555649d4fe7145badb2caf9 in 594ms, sequenceid=425, compaction requested=true 2024-11-13T10:38:35,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:35,855 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:35,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:35,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:35,855 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:35,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:35,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:35,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:35,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:35,856 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:35,856 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:35,857 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:35,857 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7f06161599ee41179493c4c1747f7826, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e12850ca5a724ddc96a743e040c2ca8a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/137d52b27690435a8173932121c8f4c6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/48731492357447cf9aade763b81ba317] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=56.1 K 2024-11-13T10:38:35,857 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:35,857 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:35,857 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:35,857 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/85e104e6f219421384aedd2e421cc8c2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e82630c85f54580a8831cc5f8e0f8e1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/222fa8e0ab7f45bd8a19df50488b18a7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/731f8f4b04484313a828f4f5fa5ae6c1] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=48.9 K 2024-11-13T10:38:35,858 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f06161599ee41179493c4c1747f7826, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731494312716 2024-11-13T10:38:35,858 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 85e104e6f219421384aedd2e421cc8c2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731494312716 2024-11-13T10:38:35,858 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e12850ca5a724ddc96a743e040c2ca8a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731494313409 2024-11-13T10:38:35,859 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e82630c85f54580a8831cc5f8e0f8e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731494313409 2024-11-13T10:38:35,859 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 137d52b27690435a8173932121c8f4c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731494314079 2024-11-13T10:38:35,859 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 222fa8e0ab7f45bd8a19df50488b18a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731494314079 2024-11-13T10:38:35,859 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48731492357447cf9aade763b81ba317, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731494315251 2024-11-13T10:38:35,860 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 731f8f4b04484313a828f4f5fa5ae6c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731494315256 2024-11-13T10:38:35,873 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:35,874 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/3ea2429ff56f4af7848d77379b4a2a0d is 50, key is test_row_0/B:col10/1731494315256/Put/seqid=0 2024-11-13T10:38:35,874 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:35,875 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/6f5958a378ae4e62a0e15848e351e8ff is 50, key is test_row_0/A:col10/1731494315256/Put/seqid=0 2024-11-13T10:38:35,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741928_1104 (size=13323) 2024-11-13T10:38:35,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741927_1103 (size=13323) 2024-11-13T10:38:35,919 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/3ea2429ff56f4af7848d77379b4a2a0d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/3ea2429ff56f4af7848d77379b4a2a0d 2024-11-13T10:38:35,927 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into 3ea2429ff56f4af7848d77379b4a2a0d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:35,927 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:35,927 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=12, startTime=1731494315855; duration=0sec 2024-11-13T10:38:35,928 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:35,928 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:35,928 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:35,932 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:35,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:38:35,932 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:35,932 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:35,932 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:35,932 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/63f4bab7dbd948aeb47d7f3d62319545, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/8bf551882b034733a8090eca311029a9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/920e298aacc14fc5b4f9a251c52bda14, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0d2fdc93e02748948d4e1072f839b9e1] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=48.9 K 2024-11-13T10:38:35,933 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f4bab7dbd948aeb47d7f3d62319545, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731494312716 2024-11-13T10:38:35,934 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bf551882b034733a8090eca311029a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731494313409 2024-11-13T10:38:35,944 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 920e298aacc14fc5b4f9a251c52bda14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731494314079 2024-11-13T10:38:35,947 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2fdc93e02748948d4e1072f839b9e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731494315256 2024-11-13T10:38:35,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/8147c76a1bf344938e6b13f6907c7508 is 50, key is test_row_0/A:col10/1731494315292/Put/seqid=0 2024-11-13T10:38:35,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494375966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494375967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494375968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,983 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:35,984 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c5fc9626b7694c1488d1b30981ebd5c7 is 50, key is test_row_0/C:col10/1731494315256/Put/seqid=0 2024-11-13T10:38:35,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494375974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494375974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741929_1105 (size=14741) 2024-11-13T10:38:36,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/8147c76a1bf344938e6b13f6907c7508 2024-11-13T10:38:36,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741930_1106 (size=13323) 2024-11-13T10:38:36,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/fcc4682cf1654a60b3882298f37cf1fb is 50, key is test_row_0/B:col10/1731494315292/Put/seqid=0 2024-11-13T10:38:36,100 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c5fc9626b7694c1488d1b30981ebd5c7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c5fc9626b7694c1488d1b30981ebd5c7 2024-11-13T10:38:36,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494376091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494376091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494376091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494376100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494376102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,109 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into c5fc9626b7694c1488d1b30981ebd5c7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:36,109 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:36,109 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=12, startTime=1731494315856; duration=0sec 2024-11-13T10:38:36,109 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:36,109 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:36,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741931_1107 (size=12301) 2024-11-13T10:38:36,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/fcc4682cf1654a60b3882298f37cf1fb 2024-11-13T10:38:36,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/354f81cb59e7400b976cd29f2bcdf3be is 50, key is test_row_0/C:col10/1731494315292/Put/seqid=0 2024-11-13T10:38:36,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741932_1108 (size=12301) 2024-11-13T10:38:36,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/354f81cb59e7400b976cd29f2bcdf3be 2024-11-13T10:38:36,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/8147c76a1bf344938e6b13f6907c7508 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8147c76a1bf344938e6b13f6907c7508 2024-11-13T10:38:36,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8147c76a1bf344938e6b13f6907c7508, entries=200, sequenceid=451, filesize=14.4 K 2024-11-13T10:38:36,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/fcc4682cf1654a60b3882298f37cf1fb as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fcc4682cf1654a60b3882298f37cf1fb 2024-11-13T10:38:36,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fcc4682cf1654a60b3882298f37cf1fb, entries=150, sequenceid=451, filesize=12.0 K 2024-11-13T10:38:36,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/354f81cb59e7400b976cd29f2bcdf3be as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/354f81cb59e7400b976cd29f2bcdf3be 2024-11-13T10:38:36,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/354f81cb59e7400b976cd29f2bcdf3be, entries=150, sequenceid=451, filesize=12.0 K 2024-11-13T10:38:36,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 0704f3508555649d4fe7145badb2caf9 in 354ms, sequenceid=451, compaction requested=false 2024-11-13T10:38:36,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:36,312 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/6f5958a378ae4e62a0e15848e351e8ff as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/6f5958a378ae4e62a0e15848e351e8ff 2024-11-13T10:38:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:36,319 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:36,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T10:38:36,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/9fb4be482f504e12afc8cf2066dc3b65 is 50, key is test_row_0/A:col10/1731494315966/Put/seqid=0 2024-11-13T10:38:36,326 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into 6f5958a378ae4e62a0e15848e351e8ff(size=13.0 K), total size for store is 27.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:36,327 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:36,327 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=12, startTime=1731494315855; duration=0sec 2024-11-13T10:38:36,327 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:36,327 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:36,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741933_1109 (size=12301) 2024-11-13T10:38:36,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/9fb4be482f504e12afc8cf2066dc3b65 2024-11-13T10:38:36,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/901589fe41f4405ba807ee8b316bc2eb is 50, key is test_row_0/B:col10/1731494315966/Put/seqid=0 2024-11-13T10:38:36,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494376373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494376375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494376377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741934_1110 (size=12301) 2024-11-13T10:38:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494376378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494376383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/901589fe41f4405ba807ee8b316bc2eb 2024-11-13T10:38:36,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/4ed63e8173844533b499dbf085f4c94a is 50, key is test_row_0/C:col10/1731494315966/Put/seqid=0 2024-11-13T10:38:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T10:38:36,429 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:36,430 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-13T10:38:36,432 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T10:38:36,440 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:36,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:36,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741935_1111 (size=12301) 2024-11-13T10:38:36,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/4ed63e8173844533b499dbf085f4c94a 2024-11-13T10:38:36,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/9fb4be482f504e12afc8cf2066dc3b65 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/9fb4be482f504e12afc8cf2066dc3b65 2024-11-13T10:38:36,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494376487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494376487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494376487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494376497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494376497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/9fb4be482f504e12afc8cf2066dc3b65, entries=150, sequenceid=468, filesize=12.0 K 2024-11-13T10:38:36,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/901589fe41f4405ba807ee8b316bc2eb as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/901589fe41f4405ba807ee8b316bc2eb 2024-11-13T10:38:36,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/901589fe41f4405ba807ee8b316bc2eb, entries=150, sequenceid=468, filesize=12.0 K 2024-11-13T10:38:36,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/4ed63e8173844533b499dbf085f4c94a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/4ed63e8173844533b499dbf085f4c94a 2024-11-13T10:38:36,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/4ed63e8173844533b499dbf085f4c94a, entries=150, sequenceid=468, filesize=12.0 K 2024-11-13T10:38:36,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 0704f3508555649d4fe7145badb2caf9 in 228ms, sequenceid=468, compaction requested=true 2024-11-13T10:38:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:36,547 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:36,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:38:36,548 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:36,549 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:36,549 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:36,549 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:36,549 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/6f5958a378ae4e62a0e15848e351e8ff, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8147c76a1bf344938e6b13f6907c7508, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/9fb4be482f504e12afc8cf2066dc3b65] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=39.4 K 2024-11-13T10:38:36,550 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f5958a378ae4e62a0e15848e351e8ff, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731494315256 2024-11-13T10:38:36,552 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:36,552 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8147c76a1bf344938e6b13f6907c7508, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731494315282 2024-11-13T10:38:36,552 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:36,552 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:36,552 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/3ea2429ff56f4af7848d77379b4a2a0d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fcc4682cf1654a60b3882298f37cf1fb, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/901589fe41f4405ba807ee8b316bc2eb] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=37.0 K 2024-11-13T10:38:36,552 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fb4be482f504e12afc8cf2066dc3b65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731494315965 2024-11-13T10:38:36,553 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ea2429ff56f4af7848d77379b4a2a0d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731494315256 2024-11-13T10:38:36,553 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc4682cf1654a60b3882298f37cf1fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731494315292 2024-11-13T10:38:36,554 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 901589fe41f4405ba807ee8b316bc2eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731494315965 2024-11-13T10:38:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T10:38:36,572 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:36,572 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/bc3f9ca6898246c38f9fc2e40a9ec30e is 50, key is test_row_0/B:col10/1731494315966/Put/seqid=0 2024-11-13T10:38:36,580 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#99 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:36,581 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/f50b03f0101a4d59b75334f993ca9d07 is 50, key is test_row_0/A:col10/1731494315966/Put/seqid=0 2024-11-13T10:38:36,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T10:38:36,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:36,600 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T10:38:36,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/2971a8e707934ba38c1be77d90209f78 is 50, key is test_row_0/A:col10/1731494316371/Put/seqid=0 2024-11-13T10:38:36,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741937_1113 (size=13425) 2024-11-13T10:38:36,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741936_1112 (size=13425) 2024-11-13T10:38:36,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741938_1114 (size=12301) 2024-11-13T10:38:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:36,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:36,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494376733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494376734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494376736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494376736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494376742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T10:38:36,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494376847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494376847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494376857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494376861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494376862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,039 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/f50b03f0101a4d59b75334f993ca9d07 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f50b03f0101a4d59b75334f993ca9d07 2024-11-13T10:38:37,053 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/bc3f9ca6898246c38f9fc2e40a9ec30e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/bc3f9ca6898246c38f9fc2e40a9ec30e 2024-11-13T10:38:37,054 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into f50b03f0101a4d59b75334f993ca9d07(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:37,054 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:37,054 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=13, startTime=1731494316547; duration=0sec 2024-11-13T10:38:37,055 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:37,055 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:37,055 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:37,059 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:37,059 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:37,059 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:37,059 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c5fc9626b7694c1488d1b30981ebd5c7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/354f81cb59e7400b976cd29f2bcdf3be, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/4ed63e8173844533b499dbf085f4c94a] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=37.0 K 2024-11-13T10:38:37,060 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5fc9626b7694c1488d1b30981ebd5c7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731494315256 2024-11-13T10:38:37,061 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 354f81cb59e7400b976cd29f2bcdf3be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731494315292 2024-11-13T10:38:37,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494377057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,066 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ed63e8173844533b499dbf085f4c94a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731494315965 2024-11-13T10:38:37,066 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into bc3f9ca6898246c38f9fc2e40a9ec30e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:37,067 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:37,067 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=13, startTime=1731494316547; duration=0sec 2024-11-13T10:38:37,067 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:37,067 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:37,070 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/2971a8e707934ba38c1be77d90209f78 2024-11-13T10:38:37,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494377068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494377068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494377070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T10:38:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494377070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,091 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:37,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/acfa2d9f20534cc8ba54aaf5f489420e is 50, key is test_row_0/B:col10/1731494316371/Put/seqid=0 2024-11-13T10:38:37,093 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/581f31c108e0434f865e10532f290fb8 is 50, key is test_row_0/C:col10/1731494315966/Put/seqid=0 2024-11-13T10:38:37,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741940_1116 (size=13425) 2024-11-13T10:38:37,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741939_1115 (size=12301) 2024-11-13T10:38:37,118 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/acfa2d9f20534cc8ba54aaf5f489420e 2024-11-13T10:38:37,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/93cb05e51ec1463798be4be006bd9bff is 50, key is test_row_0/C:col10/1731494316371/Put/seqid=0 2024-11-13T10:38:37,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741941_1117 (size=12301) 2024-11-13T10:38:37,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494377367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494377377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494377377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494377377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494377387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,525 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/581f31c108e0434f865e10532f290fb8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/581f31c108e0434f865e10532f290fb8 2024-11-13T10:38:37,532 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into 581f31c108e0434f865e10532f290fb8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:37,532 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:37,532 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=13, startTime=1731494316547; duration=0sec 2024-11-13T10:38:37,532 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:37,532 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:37,573 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/93cb05e51ec1463798be4be006bd9bff 2024-11-13T10:38:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T10:38:37,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/2971a8e707934ba38c1be77d90209f78 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/2971a8e707934ba38c1be77d90209f78 2024-11-13T10:38:37,594 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/2971a8e707934ba38c1be77d90209f78, entries=150, sequenceid=489, filesize=12.0 K 2024-11-13T10:38:37,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/acfa2d9f20534cc8ba54aaf5f489420e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/acfa2d9f20534cc8ba54aaf5f489420e 2024-11-13T10:38:37,602 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/acfa2d9f20534cc8ba54aaf5f489420e, entries=150, sequenceid=489, filesize=12.0 K 2024-11-13T10:38:37,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/93cb05e51ec1463798be4be006bd9bff as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/93cb05e51ec1463798be4be006bd9bff 2024-11-13T10:38:37,632 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/93cb05e51ec1463798be4be006bd9bff, entries=150, sequenceid=489, filesize=12.0 K 2024-11-13T10:38:37,633 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 0704f3508555649d4fe7145badb2caf9 in 1033ms, sequenceid=489, compaction requested=false 2024-11-13T10:38:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-13T10:38:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-13T10:38:37,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T10:38:37,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1960 sec 2024-11-13T10:38:37,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.2110 sec 2024-11-13T10:38:37,883 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:37,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T10:38:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:37,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/56fe2dafc020446cb3b76775b6c032a1 is 50, key is test_row_0/A:col10/1731494316727/Put/seqid=0 2024-11-13T10:38:37,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494377929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741942_1118 (size=14741) 2024-11-13T10:38:37,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/56fe2dafc020446cb3b76775b6c032a1 2024-11-13T10:38:37,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494377938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494377939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494377941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494377941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:37,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0984332093cb4c2dae347afaf8dbcbe0 is 50, key is test_row_0/B:col10/1731494316727/Put/seqid=0 2024-11-13T10:38:38,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741943_1119 (size=12301) 2024-11-13T10:38:38,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0984332093cb4c2dae347afaf8dbcbe0 2024-11-13T10:38:38,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494378048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/9a2019643b7f4059b0ecc282379c6d5a is 50, key is test_row_0/C:col10/1731494316727/Put/seqid=0 2024-11-13T10:38:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494378059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494378070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494378070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494378070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741944_1120 (size=12301) 2024-11-13T10:38:38,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494378260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494378268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494378277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494378281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494378277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/9a2019643b7f4059b0ecc282379c6d5a 2024-11-13T10:38:38,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/56fe2dafc020446cb3b76775b6c032a1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/56fe2dafc020446cb3b76775b6c032a1 2024-11-13T10:38:38,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/56fe2dafc020446cb3b76775b6c032a1, entries=200, sequenceid=510, filesize=14.4 K 2024-11-13T10:38:38,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/0984332093cb4c2dae347afaf8dbcbe0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0984332093cb4c2dae347afaf8dbcbe0 2024-11-13T10:38:38,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0984332093cb4c2dae347afaf8dbcbe0, entries=150, sequenceid=510, filesize=12.0 K 2024-11-13T10:38:38,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/9a2019643b7f4059b0ecc282379c6d5a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9a2019643b7f4059b0ecc282379c6d5a 2024-11-13T10:38:38,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9a2019643b7f4059b0ecc282379c6d5a, entries=150, sequenceid=510, filesize=12.0 K 2024-11-13T10:38:38,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 0704f3508555649d4fe7145badb2caf9 in 664ms, sequenceid=510, compaction requested=true 2024-11-13T10:38:38,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:38,548 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:38,548 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:38,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:38,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:38,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0704f3508555649d4fe7145badb2caf9:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:38,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:38,553 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:38,553 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:38,553 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/B is initiating minor compaction (all files) 2024-11-13T10:38:38,553 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/A is initiating minor compaction (all files) 2024-11-13T10:38:38,553 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/B in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:38,553 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/A in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:38,553 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/bc3f9ca6898246c38f9fc2e40a9ec30e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/acfa2d9f20534cc8ba54aaf5f489420e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0984332093cb4c2dae347afaf8dbcbe0] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=37.1 K 2024-11-13T10:38:38,553 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f50b03f0101a4d59b75334f993ca9d07, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/2971a8e707934ba38c1be77d90209f78, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/56fe2dafc020446cb3b76775b6c032a1] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=39.5 K 2024-11-13T10:38:38,554 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting bc3f9ca6898246c38f9fc2e40a9ec30e, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731494315965 2024-11-13T10:38:38,554 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f50b03f0101a4d59b75334f993ca9d07, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731494315965 2024-11-13T10:38:38,554 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting acfa2d9f20534cc8ba54aaf5f489420e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731494316361 2024-11-13T10:38:38,555 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2971a8e707934ba38c1be77d90209f78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731494316361 2024-11-13T10:38:38,555 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0984332093cb4c2dae347afaf8dbcbe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731494316712 2024-11-13T10:38:38,555 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56fe2dafc020446cb3b76775b6c032a1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731494316712 2024-11-13T10:38:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:38,571 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:38,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T10:38:38,586 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#A#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:38,587 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/b1bfeebd3957465e9feb8d419001a887 is 50, key is test_row_0/A:col10/1731494316727/Put/seqid=0 2024-11-13T10:38:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T10:38:38,597 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:38,598 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-13T10:38:38,601 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:38,601 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:38,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T10:38:38,606 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#B#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:38,607 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/119763bf3d324aedae58f055b240b77b is 50, key is test_row_0/B:col10/1731494316727/Put/seqid=0 2024-11-13T10:38:38,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/ea65ba28cca84c4f989f259b4a4ed3ba is 50, key is test_row_0/A:col10/1731494318568/Put/seqid=0 2024-11-13T10:38:38,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494378615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494378618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494378618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494378619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494378620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741945_1121 (size=13527) 2024-11-13T10:38:38,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741946_1122 (size=13527) 2024-11-13T10:38:38,676 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/b1bfeebd3957465e9feb8d419001a887 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/b1bfeebd3957465e9feb8d419001a887 2024-11-13T10:38:38,680 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/119763bf3d324aedae58f055b240b77b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/119763bf3d324aedae58f055b240b77b 2024-11-13T10:38:38,684 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/A of 0704f3508555649d4fe7145badb2caf9 into b1bfeebd3957465e9feb8d419001a887(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:38,684 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:38,685 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/A, priority=13, startTime=1731494318548; duration=0sec 2024-11-13T10:38:38,685 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:38,685 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:A 2024-11-13T10:38:38,685 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:38,687 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:38,687 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0704f3508555649d4fe7145badb2caf9/C is initiating minor compaction (all files) 2024-11-13T10:38:38,687 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0704f3508555649d4fe7145badb2caf9/C in TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:38,687 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/581f31c108e0434f865e10532f290fb8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/93cb05e51ec1463798be4be006bd9bff, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9a2019643b7f4059b0ecc282379c6d5a] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp, totalSize=37.1 K 2024-11-13T10:38:38,688 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 581f31c108e0434f865e10532f290fb8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731494315965 2024-11-13T10:38:38,688 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93cb05e51ec1463798be4be006bd9bff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731494316361 2024-11-13T10:38:38,689 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/B of 0704f3508555649d4fe7145badb2caf9 into 119763bf3d324aedae58f055b240b77b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:38,689 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:38,689 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/B, priority=13, startTime=1731494318548; duration=0sec 2024-11-13T10:38:38,689 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a2019643b7f4059b0ecc282379c6d5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731494316712 2024-11-13T10:38:38,689 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:38,689 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:B 2024-11-13T10:38:38,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741947_1123 (size=14741) 2024-11-13T10:38:38,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/ea65ba28cca84c4f989f259b4a4ed3ba 2024-11-13T10:38:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T10:38:38,714 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0704f3508555649d4fe7145badb2caf9#C#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:38,715 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/cb09264804264902b6c0da9c24368b6e is 50, key is test_row_0/C:col10/1731494316727/Put/seqid=0 2024-11-13T10:38:38,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/c0adc12adc094204aa262b94df2b36ed is 50, key is test_row_0/B:col10/1731494318568/Put/seqid=0 2024-11-13T10:38:38,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494378742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494378742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494378742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494378742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741948_1124 (size=13527) 2024-11-13T10:38:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494378742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T10:38:38,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:38,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:38,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:38,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:38,762 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/cb09264804264902b6c0da9c24368b6e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/cb09264804264902b6c0da9c24368b6e 2024-11-13T10:38:38,771 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0704f3508555649d4fe7145badb2caf9/C of 0704f3508555649d4fe7145badb2caf9 into cb09264804264902b6c0da9c24368b6e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:38,771 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:38,771 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., storeName=0704f3508555649d4fe7145badb2caf9/C, priority=13, startTime=1731494318549; duration=0sec 2024-11-13T10:38:38,772 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:38,772 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0704f3508555649d4fe7145badb2caf9:C 2024-11-13T10:38:38,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741949_1125 (size=12301) 2024-11-13T10:38:38,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T10:38:38,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:38,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:38,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:38,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:38,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T10:38:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:38,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494378957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494378957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494378957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494378958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:38,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494378958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:39,020 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-13T10:38:39,020 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:38:39,028 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-13T10:38:39,028 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:38:39,028 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-13T10:38:39,028 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:38:39,029 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,029 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,029 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,029 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,030 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,031 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:38:39,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:38:39,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:38:39,041 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-13T10:38:39,041 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:38:39,041 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,041 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:38:39,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T10:38:39,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:39,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:39,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:39,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T10:38:39,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:39,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:39,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:39,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T10:38:39,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/c0adc12adc094204aa262b94df2b36ed 2024-11-13T10:38:39,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c5d003262ae14df09cb8bd39dbece0f8 is 50, key is test_row_0/C:col10/1731494318568/Put/seqid=0 2024-11-13T10:38:39,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741950_1126 (size=12301) 2024-11-13T10:38:39,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:39,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:39,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54514 deadline: 1731494379267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54448 deadline: 1731494379267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54468 deadline: 1731494379267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:39,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:39,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54520 deadline: 1731494379267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54492 deadline: 1731494379267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:38:39,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T10:38:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:39,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T10:38:39,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:39,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:39,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c5d003262ae14df09cb8bd39dbece0f8 2024-11-13T10:38:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/ea65ba28cca84c4f989f259b4a4ed3ba as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/ea65ba28cca84c4f989f259b4a4ed3ba 2024-11-13T10:38:39,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/ea65ba28cca84c4f989f259b4a4ed3ba, entries=200, sequenceid=530, filesize=14.4 K 2024-11-13T10:38:39,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/c0adc12adc094204aa262b94df2b36ed as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/c0adc12adc094204aa262b94df2b36ed 2024-11-13T10:38:39,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/c0adc12adc094204aa262b94df2b36ed, entries=150, sequenceid=530, filesize=12.0 K 2024-11-13T10:38:39,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/c5d003262ae14df09cb8bd39dbece0f8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c5d003262ae14df09cb8bd39dbece0f8 2024-11-13T10:38:39,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c5d003262ae14df09cb8bd39dbece0f8, entries=150, sequenceid=530, filesize=12.0 K 2024-11-13T10:38:39,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 0704f3508555649d4fe7145badb2caf9 in 1110ms, sequenceid=530, compaction requested=false 2024-11-13T10:38:39,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:39,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T10:38:39,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:39,692 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T10:38:39,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/7d0c6ca0e7e64f6ab78c028e10732d21 is 50, key is test_row_0/A:col10/1731494318617/Put/seqid=0 2024-11-13T10:38:39,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741951_1127 (size=12301) 2024-11-13T10:38:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T10:38:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:39,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. as already flushing 2024-11-13T10:38:39,819 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-13T10:38:39,819 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-13T10:38:39,819 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-13T10:38:39,819 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:38:39,819 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:38:39,819 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,819 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,819 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:38:39,819 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,819 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,819 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,819 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:38:39,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:38:39,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:38:39,822 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-13T10:38:39,822 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-13T10:38:39,822 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:38:39,822 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,822 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:38:39,822 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,822 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:38:39,822 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:39,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:38:40,104 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/7d0c6ca0e7e64f6ab78c028e10732d21 2024-11-13T10:38:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/df7de926d2ec46aaa4654f5809c6cb48 is 50, key is test_row_0/B:col10/1731494318617/Put/seqid=0 2024-11-13T10:38:40,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741952_1128 (size=12301) 2024-11-13T10:38:40,517 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/df7de926d2ec46aaa4654f5809c6cb48 2024-11-13T10:38:40,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/662b52f9572e495f9113e420ed6f2469 is 50, key is test_row_0/C:col10/1731494318617/Put/seqid=0 2024-11-13T10:38:40,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741953_1129 (size=12301) 2024-11-13T10:38:40,540 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/662b52f9572e495f9113e420ed6f2469 2024-11-13T10:38:40,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/7d0c6ca0e7e64f6ab78c028e10732d21 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7d0c6ca0e7e64f6ab78c028e10732d21 2024-11-13T10:38:40,552 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7d0c6ca0e7e64f6ab78c028e10732d21, entries=150, sequenceid=550, filesize=12.0 K 2024-11-13T10:38:40,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/df7de926d2ec46aaa4654f5809c6cb48 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/df7de926d2ec46aaa4654f5809c6cb48 2024-11-13T10:38:40,563 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/df7de926d2ec46aaa4654f5809c6cb48, entries=150, sequenceid=550, filesize=12.0 K 2024-11-13T10:38:40,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/662b52f9572e495f9113e420ed6f2469 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/662b52f9572e495f9113e420ed6f2469 2024-11-13T10:38:40,570 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/662b52f9572e495f9113e420ed6f2469, entries=150, sequenceid=550, filesize=12.0 K 2024-11-13T10:38:40,571 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=33.54 KB/34350 for 0704f3508555649d4fe7145badb2caf9 in 879ms, sequenceid=550, compaction requested=true 2024-11-13T10:38:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 0704f3508555649d4fe7145badb2caf9: 2024-11-13T10:38:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-13T10:38:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-13T10:38:40,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-13T10:38:40,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9710 sec 2024-11-13T10:38:40,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.9770 sec 2024-11-13T10:38:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T10:38:40,768 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:40,768 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T10:38:40,768 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-13T10:38:40,768 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-13T10:38:40,768 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-13T10:38:40,768 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-13T10:38:40,768 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-13T10:38:40,768 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T10:38:40,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2074 2024-11-13T10:38:40,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2027 2024-11-13T10:38:40,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T10:38:40,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1013 2024-11-13T10:38:40,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3038 rows 2024-11-13T10:38:40,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1019 2024-11-13T10:38:40,769 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3056 rows 2024-11-13T10:38:40,769 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T10:38:40,769 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T10:38:40,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:40,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:40,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:38:40,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T10:38:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T10:38:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T10:38:40,785 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494320785"}]},"ts":"1731494320785"} 2024-11-13T10:38:40,788 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T10:38:40,788 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T10:38:40,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T10:38:40,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0704f3508555649d4fe7145badb2caf9, UNASSIGN}] 2024-11-13T10:38:40,797 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0704f3508555649d4fe7145badb2caf9, UNASSIGN 2024-11-13T10:38:40,798 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=0704f3508555649d4fe7145badb2caf9, regionState=CLOSING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:38:40,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0704f3508555649d4fe7145badb2caf9, UNASSIGN because future has completed 2024-11-13T10:38:40,801 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T10:38:40,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:38:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T10:38:40,960 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:40,960 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T10:38:40,960 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 0704f3508555649d4fe7145badb2caf9, disabling compactions & flushes 2024-11-13T10:38:40,960 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:40,961 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:40,961 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. after waiting 0 ms 2024-11-13T10:38:40,961 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:40,961 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 0704f3508555649d4fe7145badb2caf9 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T10:38:40,968 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/985fa8850a8d445981621fced4f0f6a9 is 50, key is test_row_0/A:col10/1731494319819/Put/seqid=0 2024-11-13T10:38:40,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741954_1130 (size=12301) 2024-11-13T10:38:41,029 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T10:38:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T10:38:41,376 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/985fa8850a8d445981621fced4f0f6a9 2024-11-13T10:38:41,385 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/e44a5b581fae45f79a405251fd805080 is 50, key is test_row_0/B:col10/1731494319819/Put/seqid=0 2024-11-13T10:38:41,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741955_1131 (size=12301) 2024-11-13T10:38:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T10:38:41,790 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/e44a5b581fae45f79a405251fd805080 2024-11-13T10:38:41,799 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/351f5bd07c5d424db35b6663bc36026b is 50, key is test_row_0/C:col10/1731494319819/Put/seqid=0 2024-11-13T10:38:41,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741956_1132 (size=12301) 2024-11-13T10:38:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T10:38:42,207 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/351f5bd07c5d424db35b6663bc36026b 2024-11-13T10:38:42,213 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/A/985fa8850a8d445981621fced4f0f6a9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/985fa8850a8d445981621fced4f0f6a9 2024-11-13T10:38:42,218 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/985fa8850a8d445981621fced4f0f6a9, entries=150, sequenceid=558, filesize=12.0 K 2024-11-13T10:38:42,219 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/B/e44a5b581fae45f79a405251fd805080 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/e44a5b581fae45f79a405251fd805080 2024-11-13T10:38:42,225 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/e44a5b581fae45f79a405251fd805080, entries=150, sequenceid=558, filesize=12.0 K 2024-11-13T10:38:42,226 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/.tmp/C/351f5bd07c5d424db35b6663bc36026b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/351f5bd07c5d424db35b6663bc36026b 2024-11-13T10:38:42,231 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/351f5bd07c5d424db35b6663bc36026b, entries=150, sequenceid=558, filesize=12.0 K 2024-11-13T10:38:42,232 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0704f3508555649d4fe7145badb2caf9 in 1271ms, sequenceid=558, compaction requested=true 2024-11-13T10:38:42,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8e2b205d2b0d4874b750b7cfc50ebdc2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0c3355dd17e4d96a1507fccf6380a64, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4b6e5d15a8d645349672b710925f5fad, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/cd2454d0607c4ee6a5953bc5bc796cb5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0edaa5f598e413693a38e29bacb83df, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/181ce2dbe09e4772b8387b95223c77a5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5de868ed4eb24458868ece58321e576a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f55db0272c46467abb577fc8da72e296, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0a2be93b1ea443185e78104c563b2f1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/fa3c1a217d714bcbb3c58fa4c2522122, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/6cdce903582f4440b743aa30afe5668d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/739e838c37c24d55b2606986f81026dd, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d3b4ad150f794cbf808a1eafd7ee2dd2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/de18b68979cb4867b2c3622cc84c77a0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5377abc56f744d8eae001894d6fc4ed8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/eb1b5f2dde104f82a205900394fbef44, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/bde13133d3de462bb8833f3df592695d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4eea08445c2e4a0d97fe453668ff88b0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/24e2ec08f9b94d8dbc190f7034d01235, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/17eb824552cc45169b00a0fa710f7257, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f2a5c99ac18b402197b067a4f18022e6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d051f0fe35a64dec9a80360d10e442ad, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7258074b175d4d6da9c6d2df3cd31f39, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4155a5b633df4588b21c81f341459785, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/c4657ca606644e58b58058b31035f834, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7f06161599ee41179493c4c1747f7826, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8d3651ad1fe4447899a55efe29c195f7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e12850ca5a724ddc96a743e040c2ca8a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/137d52b27690435a8173932121c8f4c6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/48731492357447cf9aade763b81ba317, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/6f5958a378ae4e62a0e15848e351e8ff, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8147c76a1bf344938e6b13f6907c7508, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f50b03f0101a4d59b75334f993ca9d07, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/9fb4be482f504e12afc8cf2066dc3b65, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/2971a8e707934ba38c1be77d90209f78, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/56fe2dafc020446cb3b76775b6c032a1] to archive 2024-11-13T10:38:42,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:38:42,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8e2b205d2b0d4874b750b7cfc50ebdc2 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8e2b205d2b0d4874b750b7cfc50ebdc2 2024-11-13T10:38:42,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0c3355dd17e4d96a1507fccf6380a64 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0c3355dd17e4d96a1507fccf6380a64 2024-11-13T10:38:42,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4b6e5d15a8d645349672b710925f5fad to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4b6e5d15a8d645349672b710925f5fad 2024-11-13T10:38:42,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/cd2454d0607c4ee6a5953bc5bc796cb5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/cd2454d0607c4ee6a5953bc5bc796cb5 2024-11-13T10:38:42,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0edaa5f598e413693a38e29bacb83df to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0edaa5f598e413693a38e29bacb83df 2024-11-13T10:38:42,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/181ce2dbe09e4772b8387b95223c77a5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/181ce2dbe09e4772b8387b95223c77a5 2024-11-13T10:38:42,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5de868ed4eb24458868ece58321e576a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5de868ed4eb24458868ece58321e576a 2024-11-13T10:38:42,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f55db0272c46467abb577fc8da72e296 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f55db0272c46467abb577fc8da72e296 2024-11-13T10:38:42,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0a2be93b1ea443185e78104c563b2f1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e0a2be93b1ea443185e78104c563b2f1 2024-11-13T10:38:42,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/fa3c1a217d714bcbb3c58fa4c2522122 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/fa3c1a217d714bcbb3c58fa4c2522122 2024-11-13T10:38:42,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/6cdce903582f4440b743aa30afe5668d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/6cdce903582f4440b743aa30afe5668d 2024-11-13T10:38:42,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/739e838c37c24d55b2606986f81026dd to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/739e838c37c24d55b2606986f81026dd 2024-11-13T10:38:42,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d3b4ad150f794cbf808a1eafd7ee2dd2 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d3b4ad150f794cbf808a1eafd7ee2dd2 2024-11-13T10:38:42,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/de18b68979cb4867b2c3622cc84c77a0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/de18b68979cb4867b2c3622cc84c77a0 2024-11-13T10:38:42,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5377abc56f744d8eae001894d6fc4ed8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/5377abc56f744d8eae001894d6fc4ed8 2024-11-13T10:38:42,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/eb1b5f2dde104f82a205900394fbef44 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/eb1b5f2dde104f82a205900394fbef44 2024-11-13T10:38:42,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/bde13133d3de462bb8833f3df592695d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/bde13133d3de462bb8833f3df592695d 2024-11-13T10:38:42,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4eea08445c2e4a0d97fe453668ff88b0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4eea08445c2e4a0d97fe453668ff88b0 2024-11-13T10:38:42,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/24e2ec08f9b94d8dbc190f7034d01235 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/24e2ec08f9b94d8dbc190f7034d01235 2024-11-13T10:38:42,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/17eb824552cc45169b00a0fa710f7257 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/17eb824552cc45169b00a0fa710f7257 2024-11-13T10:38:42,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f2a5c99ac18b402197b067a4f18022e6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f2a5c99ac18b402197b067a4f18022e6 2024-11-13T10:38:42,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d051f0fe35a64dec9a80360d10e442ad to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/d051f0fe35a64dec9a80360d10e442ad 2024-11-13T10:38:42,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7258074b175d4d6da9c6d2df3cd31f39 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7258074b175d4d6da9c6d2df3cd31f39 2024-11-13T10:38:42,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4155a5b633df4588b21c81f341459785 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/4155a5b633df4588b21c81f341459785 2024-11-13T10:38:42,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/c4657ca606644e58b58058b31035f834 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/c4657ca606644e58b58058b31035f834 2024-11-13T10:38:42,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7f06161599ee41179493c4c1747f7826 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7f06161599ee41179493c4c1747f7826 2024-11-13T10:38:42,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8d3651ad1fe4447899a55efe29c195f7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8d3651ad1fe4447899a55efe29c195f7 2024-11-13T10:38:42,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e12850ca5a724ddc96a743e040c2ca8a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/e12850ca5a724ddc96a743e040c2ca8a 2024-11-13T10:38:42,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/137d52b27690435a8173932121c8f4c6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/137d52b27690435a8173932121c8f4c6 2024-11-13T10:38:42,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/48731492357447cf9aade763b81ba317 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/48731492357447cf9aade763b81ba317 2024-11-13T10:38:42,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/6f5958a378ae4e62a0e15848e351e8ff to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/6f5958a378ae4e62a0e15848e351e8ff 2024-11-13T10:38:42,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8147c76a1bf344938e6b13f6907c7508 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/8147c76a1bf344938e6b13f6907c7508 2024-11-13T10:38:42,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f50b03f0101a4d59b75334f993ca9d07 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/f50b03f0101a4d59b75334f993ca9d07 2024-11-13T10:38:42,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/9fb4be482f504e12afc8cf2066dc3b65 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/9fb4be482f504e12afc8cf2066dc3b65 2024-11-13T10:38:42,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/2971a8e707934ba38c1be77d90209f78 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/2971a8e707934ba38c1be77d90209f78 2024-11-13T10:38:42,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/56fe2dafc020446cb3b76775b6c032a1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/56fe2dafc020446cb3b76775b6c032a1 2024-11-13T10:38:42,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e2903e912724ddb9aa0a4bc4e38d17f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/5f806a3941734586ba8440c074e6266c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/19f482a4a2494f71a5f1583006165103, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/59ef647f86ed4116b7aff394c4a03777, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/23d90964509d4f9e888bf9c72292516a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/2642146c8a91494bba98a397fd601b4e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0170adf13ed6427e9a3eb0470e87f334, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/de0faed06291471e91639e12fddd8285, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/35ba31e5a37b4f2e84eefd56cec49c27, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/81882e752dec49989f7c31f0b0f3f0f2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4421481e65a6447e9ee1678c455a9c6f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0939772bb412413a99c62e17774d4177, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0469d336fb0f4f1db8743c61edd72888, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/a4818fa54c7e41309342266923713e30, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/c037ee151db04ebaa84a6f71eb2bf589, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/351bbf6e178146cdb024f51bbb3ec851, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/9a7a040a330b47579d53f3132278d615, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/7d1014cd1eb54edcbf44fb022ea0ea7d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0375ea0dee18406392fa9646bcd214f7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fe9aaf3c53944ea990e3cd38c02a7fb3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/af78768d2df04cb7965b8d92e2a96249, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/dab7983f9882433383fb8587ac9f629b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/d1692f4f79a44812aba1d8b0d14e5e09, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/899636249f304be09ab516feae8292a8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/24e88cdf878a413b958de0d22c6e5bcc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/85e104e6f219421384aedd2e421cc8c2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/cd23e79ce47346c4b38dbb3115670bf9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e82630c85f54580a8831cc5f8e0f8e1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/222fa8e0ab7f45bd8a19df50488b18a7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/3ea2429ff56f4af7848d77379b4a2a0d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/731f8f4b04484313a828f4f5fa5ae6c1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fcc4682cf1654a60b3882298f37cf1fb, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/bc3f9ca6898246c38f9fc2e40a9ec30e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/901589fe41f4405ba807ee8b316bc2eb, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/acfa2d9f20534cc8ba54aaf5f489420e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0984332093cb4c2dae347afaf8dbcbe0] to archive 2024-11-13T10:38:42,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:38:42,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e2903e912724ddb9aa0a4bc4e38d17f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e2903e912724ddb9aa0a4bc4e38d17f 2024-11-13T10:38:42,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/5f806a3941734586ba8440c074e6266c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/5f806a3941734586ba8440c074e6266c 2024-11-13T10:38:42,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/19f482a4a2494f71a5f1583006165103 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/19f482a4a2494f71a5f1583006165103 2024-11-13T10:38:42,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/59ef647f86ed4116b7aff394c4a03777 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/59ef647f86ed4116b7aff394c4a03777 2024-11-13T10:38:42,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/23d90964509d4f9e888bf9c72292516a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/23d90964509d4f9e888bf9c72292516a 2024-11-13T10:38:42,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/2642146c8a91494bba98a397fd601b4e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/2642146c8a91494bba98a397fd601b4e 2024-11-13T10:38:42,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0170adf13ed6427e9a3eb0470e87f334 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0170adf13ed6427e9a3eb0470e87f334 2024-11-13T10:38:42,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/de0faed06291471e91639e12fddd8285 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/de0faed06291471e91639e12fddd8285 2024-11-13T10:38:42,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/35ba31e5a37b4f2e84eefd56cec49c27 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/35ba31e5a37b4f2e84eefd56cec49c27 2024-11-13T10:38:42,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/81882e752dec49989f7c31f0b0f3f0f2 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/81882e752dec49989f7c31f0b0f3f0f2 2024-11-13T10:38:42,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4421481e65a6447e9ee1678c455a9c6f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4421481e65a6447e9ee1678c455a9c6f 2024-11-13T10:38:42,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0939772bb412413a99c62e17774d4177 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0939772bb412413a99c62e17774d4177 2024-11-13T10:38:42,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0469d336fb0f4f1db8743c61edd72888 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0469d336fb0f4f1db8743c61edd72888 2024-11-13T10:38:42,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/a4818fa54c7e41309342266923713e30 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/a4818fa54c7e41309342266923713e30 2024-11-13T10:38:42,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/c037ee151db04ebaa84a6f71eb2bf589 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/c037ee151db04ebaa84a6f71eb2bf589 2024-11-13T10:38:42,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/351bbf6e178146cdb024f51bbb3ec851 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/351bbf6e178146cdb024f51bbb3ec851 2024-11-13T10:38:42,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/9a7a040a330b47579d53f3132278d615 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/9a7a040a330b47579d53f3132278d615 2024-11-13T10:38:42,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/7d1014cd1eb54edcbf44fb022ea0ea7d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/7d1014cd1eb54edcbf44fb022ea0ea7d 2024-11-13T10:38:42,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0375ea0dee18406392fa9646bcd214f7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0375ea0dee18406392fa9646bcd214f7 2024-11-13T10:38:42,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fe9aaf3c53944ea990e3cd38c02a7fb3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fe9aaf3c53944ea990e3cd38c02a7fb3 2024-11-13T10:38:42,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/af78768d2df04cb7965b8d92e2a96249 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/af78768d2df04cb7965b8d92e2a96249 2024-11-13T10:38:42,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/dab7983f9882433383fb8587ac9f629b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/dab7983f9882433383fb8587ac9f629b 2024-11-13T10:38:42,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/d1692f4f79a44812aba1d8b0d14e5e09 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/d1692f4f79a44812aba1d8b0d14e5e09 2024-11-13T10:38:42,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/899636249f304be09ab516feae8292a8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/899636249f304be09ab516feae8292a8 2024-11-13T10:38:42,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/24e88cdf878a413b958de0d22c6e5bcc to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/24e88cdf878a413b958de0d22c6e5bcc 2024-11-13T10:38:42,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/85e104e6f219421384aedd2e421cc8c2 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/85e104e6f219421384aedd2e421cc8c2 2024-11-13T10:38:42,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/cd23e79ce47346c4b38dbb3115670bf9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/cd23e79ce47346c4b38dbb3115670bf9 2024-11-13T10:38:42,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e82630c85f54580a8831cc5f8e0f8e1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/4e82630c85f54580a8831cc5f8e0f8e1 2024-11-13T10:38:42,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/222fa8e0ab7f45bd8a19df50488b18a7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/222fa8e0ab7f45bd8a19df50488b18a7 2024-11-13T10:38:42,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/3ea2429ff56f4af7848d77379b4a2a0d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/3ea2429ff56f4af7848d77379b4a2a0d 2024-11-13T10:38:42,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/731f8f4b04484313a828f4f5fa5ae6c1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/731f8f4b04484313a828f4f5fa5ae6c1 2024-11-13T10:38:42,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fcc4682cf1654a60b3882298f37cf1fb to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/fcc4682cf1654a60b3882298f37cf1fb 2024-11-13T10:38:42,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/bc3f9ca6898246c38f9fc2e40a9ec30e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/bc3f9ca6898246c38f9fc2e40a9ec30e 2024-11-13T10:38:42,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/901589fe41f4405ba807ee8b316bc2eb to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/901589fe41f4405ba807ee8b316bc2eb 2024-11-13T10:38:42,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/acfa2d9f20534cc8ba54aaf5f489420e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/acfa2d9f20534cc8ba54aaf5f489420e 2024-11-13T10:38:42,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0984332093cb4c2dae347afaf8dbcbe0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/0984332093cb4c2dae347afaf8dbcbe0 2024-11-13T10:38:42,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0b9bcebb63f84b1b91f69705cd880524, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/fbf1873ca7694f6280f6e80ca86cdd85, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c35b7be81a344f16ad3ec9f15d099868, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e04feef815244765a6dd58b7ffdd7cf0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ec675f575be54ae0b792b74911b74d8d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/32aef205c25143b9926796ba31505239, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/81ede5f1d99b41689fb445948b8b245b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/6d0acd039f614939ab650a651f8353da, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ffe40653a67b45e1935137f3a2da93d4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/aa1ea1cb724d498e88b3d7baca314ad7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7aa5710b23754ed89bbd8d78bf726961, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9ded69e2b1454ac09fb353601acdec1e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ceb25a9a967649c38469251757fc9dbd, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/3e4cdfdf88534984a3649ac95309bf0e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7f5eda8d0a76453da5150ddb099e579d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/831d6ae0bb20465f8bf00456f2e24100, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/49308d357399458e8ba268c445f2a26d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/12d1911dd6654490a43c98323ad56edb, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ea90f7fd37724b2e969681f5a4d7bacd, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e9b3d5e4f7184b298b0fb3ffa4efda84, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c36b6b1677b2415fbccdef7b75eac90d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/075b305a72bf49cf8d79e5e2a5644eae, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c84091d1e6fb4effa18bd7c7478a2828, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/1235f26a6bbf46f5aa94b38c4a52d051, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e3947ec4f3ed44d4a9bf4af704596130, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/63f4bab7dbd948aeb47d7f3d62319545, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/75c8d002a09344e7afc7df0578b18051, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/8bf551882b034733a8090eca311029a9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/920e298aacc14fc5b4f9a251c52bda14, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c5fc9626b7694c1488d1b30981ebd5c7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0d2fdc93e02748948d4e1072f839b9e1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/354f81cb59e7400b976cd29f2bcdf3be, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/581f31c108e0434f865e10532f290fb8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/4ed63e8173844533b499dbf085f4c94a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/93cb05e51ec1463798be4be006bd9bff, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9a2019643b7f4059b0ecc282379c6d5a] to archive 2024-11-13T10:38:42,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:38:42,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0b9bcebb63f84b1b91f69705cd880524 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0b9bcebb63f84b1b91f69705cd880524 2024-11-13T10:38:42,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/fbf1873ca7694f6280f6e80ca86cdd85 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/fbf1873ca7694f6280f6e80ca86cdd85 2024-11-13T10:38:42,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c35b7be81a344f16ad3ec9f15d099868 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c35b7be81a344f16ad3ec9f15d099868 2024-11-13T10:38:42,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e04feef815244765a6dd58b7ffdd7cf0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e04feef815244765a6dd58b7ffdd7cf0 2024-11-13T10:38:42,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ec675f575be54ae0b792b74911b74d8d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ec675f575be54ae0b792b74911b74d8d 2024-11-13T10:38:42,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/32aef205c25143b9926796ba31505239 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/32aef205c25143b9926796ba31505239 2024-11-13T10:38:42,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/81ede5f1d99b41689fb445948b8b245b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/81ede5f1d99b41689fb445948b8b245b 2024-11-13T10:38:42,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/6d0acd039f614939ab650a651f8353da to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/6d0acd039f614939ab650a651f8353da 2024-11-13T10:38:42,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ffe40653a67b45e1935137f3a2da93d4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ffe40653a67b45e1935137f3a2da93d4 2024-11-13T10:38:42,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/aa1ea1cb724d498e88b3d7baca314ad7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/aa1ea1cb724d498e88b3d7baca314ad7 2024-11-13T10:38:42,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7aa5710b23754ed89bbd8d78bf726961 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7aa5710b23754ed89bbd8d78bf726961 2024-11-13T10:38:42,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9ded69e2b1454ac09fb353601acdec1e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9ded69e2b1454ac09fb353601acdec1e 2024-11-13T10:38:42,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ceb25a9a967649c38469251757fc9dbd to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ceb25a9a967649c38469251757fc9dbd 2024-11-13T10:38:42,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/3e4cdfdf88534984a3649ac95309bf0e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/3e4cdfdf88534984a3649ac95309bf0e 2024-11-13T10:38:42,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7f5eda8d0a76453da5150ddb099e579d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/7f5eda8d0a76453da5150ddb099e579d 2024-11-13T10:38:42,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/831d6ae0bb20465f8bf00456f2e24100 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/831d6ae0bb20465f8bf00456f2e24100 2024-11-13T10:38:42,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/49308d357399458e8ba268c445f2a26d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/49308d357399458e8ba268c445f2a26d 2024-11-13T10:38:42,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/12d1911dd6654490a43c98323ad56edb to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/12d1911dd6654490a43c98323ad56edb 2024-11-13T10:38:42,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ea90f7fd37724b2e969681f5a4d7bacd to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/ea90f7fd37724b2e969681f5a4d7bacd 2024-11-13T10:38:42,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e9b3d5e4f7184b298b0fb3ffa4efda84 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e9b3d5e4f7184b298b0fb3ffa4efda84 2024-11-13T10:38:42,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c36b6b1677b2415fbccdef7b75eac90d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c36b6b1677b2415fbccdef7b75eac90d 2024-11-13T10:38:42,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/075b305a72bf49cf8d79e5e2a5644eae to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/075b305a72bf49cf8d79e5e2a5644eae 2024-11-13T10:38:42,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c84091d1e6fb4effa18bd7c7478a2828 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c84091d1e6fb4effa18bd7c7478a2828 2024-11-13T10:38:42,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/1235f26a6bbf46f5aa94b38c4a52d051 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/1235f26a6bbf46f5aa94b38c4a52d051 2024-11-13T10:38:42,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e3947ec4f3ed44d4a9bf4af704596130 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/e3947ec4f3ed44d4a9bf4af704596130 2024-11-13T10:38:42,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/63f4bab7dbd948aeb47d7f3d62319545 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/63f4bab7dbd948aeb47d7f3d62319545 2024-11-13T10:38:42,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/75c8d002a09344e7afc7df0578b18051 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/75c8d002a09344e7afc7df0578b18051 2024-11-13T10:38:42,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/8bf551882b034733a8090eca311029a9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/8bf551882b034733a8090eca311029a9 2024-11-13T10:38:42,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/920e298aacc14fc5b4f9a251c52bda14 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/920e298aacc14fc5b4f9a251c52bda14 2024-11-13T10:38:42,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c5fc9626b7694c1488d1b30981ebd5c7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c5fc9626b7694c1488d1b30981ebd5c7 2024-11-13T10:38:42,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0d2fdc93e02748948d4e1072f839b9e1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/0d2fdc93e02748948d4e1072f839b9e1 2024-11-13T10:38:42,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/354f81cb59e7400b976cd29f2bcdf3be to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/354f81cb59e7400b976cd29f2bcdf3be 2024-11-13T10:38:42,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/581f31c108e0434f865e10532f290fb8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/581f31c108e0434f865e10532f290fb8 2024-11-13T10:38:42,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/4ed63e8173844533b499dbf085f4c94a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/4ed63e8173844533b499dbf085f4c94a 2024-11-13T10:38:42,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/93cb05e51ec1463798be4be006bd9bff to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/93cb05e51ec1463798be4be006bd9bff 2024-11-13T10:38:42,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9a2019643b7f4059b0ecc282379c6d5a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/9a2019643b7f4059b0ecc282379c6d5a 2024-11-13T10:38:42,473 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/recovered.edits/561.seqid, newMaxSeqId=561, maxSeqId=1 2024-11-13T10:38:42,474 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. 2024-11-13T10:38:42,474 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 0704f3508555649d4fe7145badb2caf9: Waiting for close lock at 1731494320960Running coprocessor pre-close hooks at 1731494320960Disabling compacts and flushes for region at 1731494320960Disabling writes for close at 1731494320961 (+1 ms)Obtaining lock to block concurrent updates at 1731494320961Preparing flush snapshotting stores in 0704f3508555649d4fe7145badb2caf9 at 1731494320961Finished memstore snapshotting TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731494320961Flushing stores of TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9. at 1731494320962 (+1 ms)Flushing 0704f3508555649d4fe7145badb2caf9/A: creating writer at 1731494320962Flushing 0704f3508555649d4fe7145badb2caf9/A: appending metadata at 1731494320967 (+5 ms)Flushing 0704f3508555649d4fe7145badb2caf9/A: closing flushed file at 1731494320967Flushing 0704f3508555649d4fe7145badb2caf9/B: creating writer at 1731494321381 (+414 ms)Flushing 0704f3508555649d4fe7145badb2caf9/B: appending metadata at 1731494321384 (+3 ms)Flushing 0704f3508555649d4fe7145badb2caf9/B: closing flushed file at 1731494321384Flushing 0704f3508555649d4fe7145badb2caf9/C: creating writer at 1731494321795 (+411 ms)Flushing 0704f3508555649d4fe7145badb2caf9/C: appending metadata at 1731494321799 (+4 ms)Flushing 0704f3508555649d4fe7145badb2caf9/C: closing flushed file at 1731494321799Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f9044c: reopening flushed file at 1731494322212 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@282aa9e4: reopening flushed file at 1731494322218 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2119be42: reopening flushed file at 1731494322225 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0704f3508555649d4fe7145badb2caf9 in 1271ms, sequenceid=558, compaction requested=true at 1731494322232 (+7 ms)Writing region close event to WAL at 1731494322470 (+238 ms)Running coprocessor post-close hooks at 1731494322473 (+3 ms)Closed at 1731494322474 (+1 ms) 2024-11-13T10:38:42,476 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:42,477 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=0704f3508555649d4fe7145badb2caf9, regionState=CLOSED 2024-11-13T10:38:42,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:38:42,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T10:38:42,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 0704f3508555649d4fe7145badb2caf9, server=770665a7984d,39101,1731494292956 in 1.6770 sec 2024-11-13T10:38:42,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-13T10:38:42,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0704f3508555649d4fe7145badb2caf9, UNASSIGN in 1.6860 sec 2024-11-13T10:38:42,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-13T10:38:42,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6940 sec 2024-11-13T10:38:42,487 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494322487"}]},"ts":"1731494322487"} 2024-11-13T10:38:42,489 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T10:38:42,489 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T10:38:42,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7130 sec 2024-11-13T10:38:42,624 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T10:38:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T10:38:42,937 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:42,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T10:38:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:38:42,950 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:38:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T10:38:42,952 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:38:42,957 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:42,961 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/recovered.edits] 2024-11-13T10:38:42,964 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7d0c6ca0e7e64f6ab78c028e10732d21 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/7d0c6ca0e7e64f6ab78c028e10732d21 2024-11-13T10:38:42,966 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/985fa8850a8d445981621fced4f0f6a9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/985fa8850a8d445981621fced4f0f6a9 2024-11-13T10:38:42,967 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/b1bfeebd3957465e9feb8d419001a887 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/b1bfeebd3957465e9feb8d419001a887 2024-11-13T10:38:42,968 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/ea65ba28cca84c4f989f259b4a4ed3ba to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/A/ea65ba28cca84c4f989f259b4a4ed3ba 2024-11-13T10:38:42,971 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/119763bf3d324aedae58f055b240b77b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/119763bf3d324aedae58f055b240b77b 2024-11-13T10:38:42,972 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/c0adc12adc094204aa262b94df2b36ed to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/c0adc12adc094204aa262b94df2b36ed 2024-11-13T10:38:42,973 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/df7de926d2ec46aaa4654f5809c6cb48 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/df7de926d2ec46aaa4654f5809c6cb48 2024-11-13T10:38:42,975 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/e44a5b581fae45f79a405251fd805080 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/B/e44a5b581fae45f79a405251fd805080 2024-11-13T10:38:42,977 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/351f5bd07c5d424db35b6663bc36026b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/351f5bd07c5d424db35b6663bc36026b 2024-11-13T10:38:42,979 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/662b52f9572e495f9113e420ed6f2469 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/662b52f9572e495f9113e420ed6f2469 2024-11-13T10:38:42,980 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c5d003262ae14df09cb8bd39dbece0f8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/c5d003262ae14df09cb8bd39dbece0f8 2024-11-13T10:38:42,982 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/cb09264804264902b6c0da9c24368b6e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/C/cb09264804264902b6c0da9c24368b6e 2024-11-13T10:38:42,985 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/recovered.edits/561.seqid to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9/recovered.edits/561.seqid 2024-11-13T10:38:42,985 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0704f3508555649d4fe7145badb2caf9 2024-11-13T10:38:42,985 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T10:38:42,991 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:38:42,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-13T10:38:43,003 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T10:38:43,037 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T10:38:43,039 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:38:43,039 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T10:38:43,039 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731494323039"}]},"ts":"9223372036854775807"} 2024-11-13T10:38:43,044 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T10:38:43,044 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0704f3508555649d4fe7145badb2caf9, NAME => 'TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T10:38:43,044 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T10:38:43,045 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731494323044"}]},"ts":"9223372036854775807"} 2024-11-13T10:38:43,049 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T10:38:43,050 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:38:43,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 107 msec 2024-11-13T10:38:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T10:38:43,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T10:38:43,058 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:43,075 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/770665a7984d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2019480356_22 at /127.0.0.1:40760 [Waiting for operation #264] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2019480356_22 at /127.0.0.1:40614 [Waiting for operation #279] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;770665a7984d:39101-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/770665a7984d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1917562498_22 at /127.0.0.1:40678 [Waiting for operation #213] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1000 (was 1055), ProcessCount=11 (was 12), AvailableMemoryMB=644 (was 1204) 2024-11-13T10:38:43,087 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=1000, ProcessCount=11, AvailableMemoryMB=641 2024-11-13T10:38:43,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T10:38:43,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T10:38:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T10:38:43,092 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T10:38:43,092 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:43,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-13T10:38:43,093 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T10:38:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-13T10:38:43,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741957_1133 (size=959) 2024-11-13T10:38:43,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-13T10:38:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-13T10:38:43,509 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617 2024-11-13T10:38:43,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741958_1134 (size=53) 2024-11-13T10:38:43,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:38:43,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 99adb18da1f9e813f4728ff7b8100fa3, disabling compactions & flushes 2024-11-13T10:38:43,516 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:43,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:43,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. after waiting 0 ms 2024-11-13T10:38:43,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:43,516 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:43,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 99adb18da1f9e813f4728ff7b8100fa3: Waiting for close lock at 1731494323516Disabling compacts and flushes for region at 1731494323516Disabling writes for close at 1731494323516Writing region close event to WAL at 1731494323516Closed at 1731494323516 2024-11-13T10:38:43,517 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T10:38:43,518 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731494323517"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731494323517"}]},"ts":"1731494323517"} 2024-11-13T10:38:43,520 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T10:38:43,521 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T10:38:43,521 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494323521"}]},"ts":"1731494323521"} 2024-11-13T10:38:43,522 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T10:38:43,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, ASSIGN}] 2024-11-13T10:38:43,524 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, ASSIGN 2024-11-13T10:38:43,524 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, ASSIGN; state=OFFLINE, location=770665a7984d,39101,1731494292956; forceNewPlan=false, retain=false 2024-11-13T10:38:43,675 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=99adb18da1f9e813f4728ff7b8100fa3, regionState=OPENING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:38:43,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, ASSIGN because future has completed 2024-11-13T10:38:43,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:38:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-13T10:38:43,839 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:43,839 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:38:43,840 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,840 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:38:43,840 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,840 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,842 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,844 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99adb18da1f9e813f4728ff7b8100fa3 columnFamilyName A 2024-11-13T10:38:43,844 DEBUG [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:43,844 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(327): Store=99adb18da1f9e813f4728ff7b8100fa3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:43,844 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,846 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99adb18da1f9e813f4728ff7b8100fa3 columnFamilyName B 2024-11-13T10:38:43,846 DEBUG [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:43,846 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(327): Store=99adb18da1f9e813f4728ff7b8100fa3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:43,846 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,847 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99adb18da1f9e813f4728ff7b8100fa3 columnFamilyName C 2024-11-13T10:38:43,847 DEBUG [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:43,848 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(327): Store=99adb18da1f9e813f4728ff7b8100fa3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:43,848 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,849 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,849 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,850 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,850 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,850 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:38:43,851 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,853 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T10:38:43,854 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 99adb18da1f9e813f4728ff7b8100fa3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59019769, jitterRate=-0.1205369085073471}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:38:43,854 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:43,855 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 99adb18da1f9e813f4728ff7b8100fa3: Running coprocessor pre-open hook at 1731494323840Writing region info on filesystem at 1731494323840Initializing all the Stores at 1731494323841 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494323841Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494323841Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494323841Cleaning up temporary data from old regions at 1731494323850 (+9 ms)Running coprocessor post-open hooks at 1731494323854 (+4 ms)Region opened successfully at 1731494323855 (+1 ms) 2024-11-13T10:38:43,855 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., pid=30, masterSystemTime=1731494323835 2024-11-13T10:38:43,858 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:43,858 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:43,858 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=99adb18da1f9e813f4728ff7b8100fa3, regionState=OPEN, openSeqNum=2, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:38:43,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:38:43,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T10:38:43,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 in 179 msec 2024-11-13T10:38:43,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-13T10:38:43,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, ASSIGN in 341 msec 2024-11-13T10:38:43,866 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T10:38:43,867 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494323866"}]},"ts":"1731494323866"} 2024-11-13T10:38:43,868 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T10:38:43,869 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T10:38:43,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 779 msec 2024-11-13T10:38:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-13T10:38:44,228 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:44,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328db2e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:44,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:44,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:44,233 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:44,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:44,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd55bc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:44,236 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9145e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:44,237 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:44,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:44,242 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T10:38:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 770665a7984d,44523,1731494292131 2024-11-13T10:38:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33e75818 2024-11-13T10:38:44,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T10:38:44,243 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T10:38:44,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T10:38:44,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T10:38:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T10:38:44,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741959_1135 (size=995) 2024-11-13T10:38:44,670 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T10:38:44,670 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T10:38:44,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T10:38:44,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, REOPEN/MOVE}] 2024-11-13T10:38:44,685 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, REOPEN/MOVE 2024-11-13T10:38:44,686 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=99adb18da1f9e813f4728ff7b8100fa3, regionState=CLOSING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:38:44,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, REOPEN/MOVE because future has completed 2024-11-13T10:38:44,698 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T10:38:44,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:38:44,851 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:44,852 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T10:38:44,852 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 99adb18da1f9e813f4728ff7b8100fa3, disabling compactions & flushes 2024-11-13T10:38:44,852 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:44,852 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:44,852 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. after waiting 0 ms 2024-11-13T10:38:44,852 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:44,857 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T10:38:44,858 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:44,858 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 99adb18da1f9e813f4728ff7b8100fa3: Waiting for close lock at 1731494324852Running coprocessor pre-close hooks at 1731494324852Disabling compacts and flushes for region at 1731494324852Disabling writes for close at 1731494324852Writing region close event to WAL at 1731494324852Running coprocessor post-close hooks at 1731494324858 (+6 ms)Closed at 1731494324858 2024-11-13T10:38:44,858 WARN [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 99adb18da1f9e813f4728ff7b8100fa3 to self. 2024-11-13T10:38:44,861 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:44,861 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=99adb18da1f9e813f4728ff7b8100fa3, regionState=CLOSED 2024-11-13T10:38:44,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:38:44,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-13T10:38:44,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 in 166 msec 2024-11-13T10:38:44,868 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, REOPEN/MOVE; state=CLOSED, location=770665a7984d,39101,1731494292956; forceNewPlan=false, retain=true 2024-11-13T10:38:45,018 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=99adb18da1f9e813f4728ff7b8100fa3, regionState=OPENING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, REOPEN/MOVE because future has completed 2024-11-13T10:38:45,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:38:45,177 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:45,177 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:38:45,177 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,177 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:38:45,177 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,177 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,180 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,185 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99adb18da1f9e813f4728ff7b8100fa3 columnFamilyName A 2024-11-13T10:38:45,187 DEBUG [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:45,188 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(327): Store=99adb18da1f9e813f4728ff7b8100fa3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:45,188 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,189 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99adb18da1f9e813f4728ff7b8100fa3 columnFamilyName B 2024-11-13T10:38:45,189 DEBUG [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:45,189 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(327): Store=99adb18da1f9e813f4728ff7b8100fa3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:45,189 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,190 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99adb18da1f9e813f4728ff7b8100fa3 columnFamilyName C 2024-11-13T10:38:45,190 DEBUG [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:45,191 INFO [StoreOpener-99adb18da1f9e813f4728ff7b8100fa3-1 {}] regionserver.HStore(327): Store=99adb18da1f9e813f4728ff7b8100fa3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:38:45,191 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,191 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,193 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,196 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,197 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,197 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:38:45,199 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,199 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 99adb18da1f9e813f4728ff7b8100fa3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59764163, jitterRate=-0.10944457352161407}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:38:45,200 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,201 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 99adb18da1f9e813f4728ff7b8100fa3: Running coprocessor pre-open hook at 1731494325178Writing region info on filesystem at 1731494325178Initializing all the Stores at 1731494325178Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494325178Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494325179 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494325179Cleaning up temporary data from old regions at 1731494325197 (+18 ms)Running coprocessor post-open hooks at 1731494325200 (+3 ms)Region opened successfully at 1731494325201 (+1 ms) 2024-11-13T10:38:45,204 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., pid=35, masterSystemTime=1731494325173 2024-11-13T10:38:45,206 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:45,206 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:45,207 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=99adb18da1f9e813f4728ff7b8100fa3, regionState=OPEN, openSeqNum=5, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:38:45,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-13T10:38:45,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 in 189 msec 2024-11-13T10:38:45,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-13T10:38:45,215 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, REOPEN/MOVE in 529 msec 2024-11-13T10:38:45,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-13T10:38:45,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 542 msec 2024-11-13T10:38:45,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 969 msec 2024-11-13T10:38:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T10:38:45,225 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:45,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263f8ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:45,226 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:45,227 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39244e70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:45,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:45,230 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:45,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c27ba6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:45,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:45,232 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6490be12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:45,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:45,234 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c069f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:45,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:45,237 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30abd120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:45,238 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:45,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c7e4ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:45,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:45,241 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520dbe7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:45,242 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:45,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d61935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:45,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:45,245 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:45,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6abf8b97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:45,247 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39681dd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:45,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:45,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44166989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:45,251 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6864ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:45,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:45,253 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:45,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:45,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:45,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d4c072, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:45,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:45,254 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b80d230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:45,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:45,257 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e92a410, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:45,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:45,259 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3626668d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:38:45,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:38:45,265 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:38:45,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:38:45,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:38:45,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@101321f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:38:45,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:38:45,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:38:45,267 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:38:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e4f9348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:38:45,273 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:45,274 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:45,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:45,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:38:45,281 DEBUG [Thread-669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:38:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:45,282 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:45,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:45,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:45,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-13T10:38:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T10:38:45,287 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:45,287 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:38:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:45,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:45,289 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:45,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:38:45,294 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:45,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,298 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:45,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:38:45,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:38:45,305 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:45,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:38:45,317 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:45,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:38:45,333 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:45,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:38:45,339 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:38:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:38:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:38:45,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:38:45,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:38:45,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e38981f6625848b9985b141060941eb9_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494325294/Put/seqid=0 2024-11-13T10:38:45,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494385354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494385355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494385357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494385355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494385367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T10:38:45,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741960_1136 (size=19474) 2024-11-13T10:38:45,411 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:45,420 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e38981f6625848b9985b141060941eb9_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e38981f6625848b9985b141060941eb9_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/a7faabe32a884e33a5cbb93384aa87d9, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:45,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/a7faabe32a884e33a5cbb93384aa87d9 is 175, key is test_row_0/A:col10/1731494325294/Put/seqid=0 2024-11-13T10:38:45,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T10:38:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:45,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:45,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741961_1137 (size=56733) 2024-11-13T10:38:45,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/a7faabe32a884e33a5cbb93384aa87d9 2024-11-13T10:38:45,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/d66b379e86c94199bd654457cf53bf3b is 50, key is test_row_0/B:col10/1731494325294/Put/seqid=0 2024-11-13T10:38:45,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494385487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494385487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494385487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741962_1138 (size=12001) 2024-11-13T10:38:45,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/d66b379e86c94199bd654457cf53bf3b 2024-11-13T10:38:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494385490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494385490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/0df76054bf1a4c88a10b34d3daea153d is 50, key is test_row_0/C:col10/1731494325294/Put/seqid=0 2024-11-13T10:38:45,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741963_1139 (size=12001) 2024-11-13T10:38:45,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/0df76054bf1a4c88a10b34d3daea153d 2024-11-13T10:38:45,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/a7faabe32a884e33a5cbb93384aa87d9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a7faabe32a884e33a5cbb93384aa87d9 2024-11-13T10:38:45,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a7faabe32a884e33a5cbb93384aa87d9, entries=300, sequenceid=15, filesize=55.4 K 2024-11-13T10:38:45,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/d66b379e86c94199bd654457cf53bf3b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/d66b379e86c94199bd654457cf53bf3b 2024-11-13T10:38:45,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/d66b379e86c94199bd654457cf53bf3b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T10:38:45,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/0df76054bf1a4c88a10b34d3daea153d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/0df76054bf1a4c88a10b34d3daea153d 2024-11-13T10:38:45,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/0df76054bf1a4c88a10b34d3daea153d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T10:38:45,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 99adb18da1f9e813f4728ff7b8100fa3 in 300ms, sequenceid=15, compaction requested=false 2024-11-13T10:38:45,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:45,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-13T10:38:45,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:45,599 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:38:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T10:38:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bcf3084cf76e4f79aff5eda6d4c36c56_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494325345/Put/seqid=0 2024-11-13T10:38:45,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741964_1140 (size=12154) 2024-11-13T10:38:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:45,652 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bcf3084cf76e4f79aff5eda6d4c36c56_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bcf3084cf76e4f79aff5eda6d4c36c56_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/da7a5e090f644d97a1b160e454485642, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:45,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/da7a5e090f644d97a1b160e454485642 is 175, key is test_row_0/A:col10/1731494325345/Put/seqid=0 2024-11-13T10:38:45,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741965_1141 (size=30955) 2024-11-13T10:38:45,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494385722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494385729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494385734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494385736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494385737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494385847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494385848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494385857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494385858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494385858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:45,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T10:38:46,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494386067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,075 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/da7a5e090f644d97a1b160e454485642 2024-11-13T10:38:46,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494386079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494386079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494386079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494386079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/bd7c808b0b384c919e0b14cb9618fc7c is 50, key is test_row_0/B:col10/1731494325345/Put/seqid=0 2024-11-13T10:38:46,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741966_1142 (size=12001) 2024-11-13T10:38:46,117 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/bd7c808b0b384c919e0b14cb9618fc7c 2024-11-13T10:38:46,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/1e8d5551f7474394b915ac87723837ca is 50, key is test_row_0/C:col10/1731494325345/Put/seqid=0 2024-11-13T10:38:46,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741967_1143 (size=12001) 2024-11-13T10:38:46,228 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/1e8d5551f7474394b915ac87723837ca 2024-11-13T10:38:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/da7a5e090f644d97a1b160e454485642 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/da7a5e090f644d97a1b160e454485642 2024-11-13T10:38:46,249 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/da7a5e090f644d97a1b160e454485642, entries=150, sequenceid=40, filesize=30.2 K 2024-11-13T10:38:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/bd7c808b0b384c919e0b14cb9618fc7c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/bd7c808b0b384c919e0b14cb9618fc7c 2024-11-13T10:38:46,260 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/bd7c808b0b384c919e0b14cb9618fc7c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T10:38:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/1e8d5551f7474394b915ac87723837ca as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/1e8d5551f7474394b915ac87723837ca 2024-11-13T10:38:46,275 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/1e8d5551f7474394b915ac87723837ca, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T10:38:46,277 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 99adb18da1f9e813f4728ff7b8100fa3 in 678ms, sequenceid=40, compaction requested=false 2024-11-13T10:38:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-13T10:38:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-13T10:38:46,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-13T10:38:46,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 991 msec 2024-11-13T10:38:46,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.0040 sec 2024-11-13T10:38:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:46,397 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:46,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T10:38:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-13T10:38:46,430 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:46,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-13T10:38:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T10:38:46,439 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:46,440 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:46,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:46,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ae4c098dd4b74580950d0cb8cd29546a_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494326392/Put/seqid=0 2024-11-13T10:38:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494386450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494386451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494386452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494386464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494386466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741968_1144 (size=14594) 2024-11-13T10:38:46,500 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:46,505 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ae4c098dd4b74580950d0cb8cd29546a_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ae4c098dd4b74580950d0cb8cd29546a_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:46,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/13f4b96d6bd74fb0a68a85ad68b93ad3, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:46,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/13f4b96d6bd74fb0a68a85ad68b93ad3 is 175, key is test_row_0/A:col10/1731494326392/Put/seqid=0 2024-11-13T10:38:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T10:38:46,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741969_1145 (size=39549) 2024-11-13T10:38:46,562 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/13f4b96d6bd74fb0a68a85ad68b93ad3 2024-11-13T10:38:46,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494386573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494386573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494386573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494386577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494386577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:46,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:46,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:46,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:46,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1e5742c3f9f94fb3b024a031df675603 is 50, key is test_row_0/B:col10/1731494326392/Put/seqid=0 2024-11-13T10:38:46,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741970_1146 (size=12001) 2024-11-13T10:38:46,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1e5742c3f9f94fb3b024a031df675603 2024-11-13T10:38:46,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/ce601d3e21114683b74a7d068b405ac0 is 50, key is test_row_0/C:col10/1731494326392/Put/seqid=0 2024-11-13T10:38:46,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741971_1147 (size=12001) 2024-11-13T10:38:46,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/ce601d3e21114683b74a7d068b405ac0 2024-11-13T10:38:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/13f4b96d6bd74fb0a68a85ad68b93ad3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/13f4b96d6bd74fb0a68a85ad68b93ad3 2024-11-13T10:38:46,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/13f4b96d6bd74fb0a68a85ad68b93ad3, entries=200, sequenceid=54, filesize=38.6 K 2024-11-13T10:38:46,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:46,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:46,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:46,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:46,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1e5742c3f9f94fb3b024a031df675603 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1e5742c3f9f94fb3b024a031df675603 2024-11-13T10:38:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T10:38:46,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1e5742c3f9f94fb3b024a031df675603, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T10:38:46,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/ce601d3e21114683b74a7d068b405ac0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/ce601d3e21114683b74a7d068b405ac0 2024-11-13T10:38:46,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/ce601d3e21114683b74a7d068b405ac0, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T10:38:46,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 99adb18da1f9e813f4728ff7b8100fa3 in 379ms, sequenceid=54, compaction requested=true 2024-11-13T10:38:46,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:46,776 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:46,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:46,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:46,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:46,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:46,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:46,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:46,777 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:46,782 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:46,782 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/B is initiating minor compaction (all files) 2024-11-13T10:38:46,782 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/B in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,782 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/d66b379e86c94199bd654457cf53bf3b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/bd7c808b0b384c919e0b14cb9618fc7c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1e5742c3f9f94fb3b024a031df675603] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=35.2 K 2024-11-13T10:38:46,782 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:46,783 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/A is initiating minor compaction (all files) 2024-11-13T10:38:46,783 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/A in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,783 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a7faabe32a884e33a5cbb93384aa87d9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/da7a5e090f644d97a1b160e454485642, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/13f4b96d6bd74fb0a68a85ad68b93ad3] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=124.3 K 2024-11-13T10:38:46,783 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,783 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a7faabe32a884e33a5cbb93384aa87d9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/da7a5e090f644d97a1b160e454485642, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/13f4b96d6bd74fb0a68a85ad68b93ad3] 2024-11-13T10:38:46,784 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting d66b379e86c94199bd654457cf53bf3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731494325289 2024-11-13T10:38:46,784 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7faabe32a884e33a5cbb93384aa87d9, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731494325286 2024-11-13T10:38:46,785 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting bd7c808b0b384c919e0b14cb9618fc7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731494325345 2024-11-13T10:38:46,785 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7a5e090f644d97a1b160e454485642, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731494325345 2024-11-13T10:38:46,785 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e5742c3f9f94fb3b024a031df675603, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494325731 2024-11-13T10:38:46,786 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13f4b96d6bd74fb0a68a85ad68b93ad3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494325731 2024-11-13T10:38:46,807 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:46,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:38:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:46,815 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:46,816 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1cc2af0f47544cc8830f48cd0f0142e9 is 50, key is test_row_0/B:col10/1731494326392/Put/seqid=0 2024-11-13T10:38:46,834 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:46,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494386840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494386840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494386855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494386859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494386858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,874 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411134da4f7ca461544a088a864b488e0ace7_99adb18da1f9e813f4728ff7b8100fa3 store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:46,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741972_1148 (size=12104) 2024-11-13T10:38:46,893 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411134da4f7ca461544a088a864b488e0ace7_99adb18da1f9e813f4728ff7b8100fa3, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:46,893 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134da4f7ca461544a088a864b488e0ace7_99adb18da1f9e813f4728ff7b8100fa3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:46,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ec6e602ee69a424bb1459b11905e8c7d_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494326461/Put/seqid=0 2024-11-13T10:38:46,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:46,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:46,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:46,921 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1cc2af0f47544cc8830f48cd0f0142e9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1cc2af0f47544cc8830f48cd0f0142e9 2024-11-13T10:38:46,942 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/B of 99adb18da1f9e813f4728ff7b8100fa3 into 1cc2af0f47544cc8830f48cd0f0142e9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:46,942 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:46,942 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/B, priority=13, startTime=1731494326777; duration=0sec 2024-11-13T10:38:46,943 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:46,943 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:B 2024-11-13T10:38:46,943 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:46,945 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:46,945 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/C is initiating minor compaction (all files) 2024-11-13T10:38:46,945 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/C in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:46,945 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/0df76054bf1a4c88a10b34d3daea153d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/1e8d5551f7474394b915ac87723837ca, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/ce601d3e21114683b74a7d068b405ac0] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=35.2 K 2024-11-13T10:38:46,946 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df76054bf1a4c88a10b34d3daea153d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731494325289 2024-11-13T10:38:46,946 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e8d5551f7474394b915ac87723837ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731494325345 2024-11-13T10:38:46,947 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ce601d3e21114683b74a7d068b405ac0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494325731 2024-11-13T10:38:46,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741973_1149 (size=4469) 2024-11-13T10:38:46,953 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#A#compaction#130 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:46,953 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/2b4a0ce805e6432eb2520c09ca8e5cd4 is 175, key is test_row_0/A:col10/1731494326392/Put/seqid=0 2024-11-13T10:38:46,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494386968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494386969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494386969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494386977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:46,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494386977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:46,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,000 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:47,001 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/e31606752df141beb86069b9e76f18a8 is 50, key is test_row_0/C:col10/1731494326392/Put/seqid=0 2024-11-13T10:38:47,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741974_1150 (size=17034) 2024-11-13T10:38:47,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741975_1151 (size=31058) 2024-11-13T10:38:47,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,063 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/2b4a0ce805e6432eb2520c09ca8e5cd4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2b4a0ce805e6432eb2520c09ca8e5cd4 2024-11-13T10:38:47,070 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/A of 99adb18da1f9e813f4728ff7b8100fa3 into 2b4a0ce805e6432eb2520c09ca8e5cd4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:47,070 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:47,070 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/A, priority=13, startTime=1731494326776; duration=0sec 2024-11-13T10:38:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T10:38:47,070 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:47,070 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:A 2024-11-13T10:38:47,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741976_1152 (size=12104) 2024-11-13T10:38:47,094 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/e31606752df141beb86069b9e76f18a8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e31606752df141beb86069b9e76f18a8 2024-11-13T10:38:47,112 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/C of 99adb18da1f9e813f4728ff7b8100fa3 into e31606752df141beb86069b9e76f18a8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:47,112 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:47,112 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/C, priority=13, startTime=1731494326777; duration=0sec 2024-11-13T10:38:47,113 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:47,113 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:C 2024-11-13T10:38:47,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494387207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494387207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:47,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494387208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494387209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494387209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:47,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:47,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,423 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:47,452 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ec6e602ee69a424bb1459b11905e8c7d_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ec6e602ee69a424bb1459b11905e8c7d_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:47,453 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ab7e667d0e724d1fa631a86f800f6097, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:47,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ab7e667d0e724d1fa631a86f800f6097 is 175, key is test_row_0/A:col10/1731494326461/Put/seqid=0 2024-11-13T10:38:47,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741977_1153 (size=48139) 2024-11-13T10:38:47,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494387517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:47,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494387517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494387518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494387518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494387530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:47,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T10:38:47,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:47,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:47,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:47,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,910 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ab7e667d0e724d1fa631a86f800f6097 2024-11-13T10:38:47,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/8d0fc3d180bd44dd814c4104cd39c371 is 50, key is test_row_0/B:col10/1731494326461/Put/seqid=0 2024-11-13T10:38:47,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741978_1154 (size=12001) 2024-11-13T10:38:47,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:47,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:47,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:47,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,020 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T10:38:48,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494388027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:48,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494388029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:48,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494388029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:48,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494388029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494388037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:48,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:48,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:48,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/8d0fc3d180bd44dd814c4104cd39c371 2024-11-13T10:38:48,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/59fb59b3606a422f81683387e5af346f is 50, key is test_row_0/C:col10/1731494326461/Put/seqid=0 2024-11-13T10:38:48,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741979_1155 (size=12001) 2024-11-13T10:38:48,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T10:38:48,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:48,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:48,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:48,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/59fb59b3606a422f81683387e5af346f 2024-11-13T10:38:48,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ab7e667d0e724d1fa631a86f800f6097 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ab7e667d0e724d1fa631a86f800f6097 2024-11-13T10:38:48,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ab7e667d0e724d1fa631a86f800f6097, entries=250, sequenceid=79, filesize=47.0 K 2024-11-13T10:38:48,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/8d0fc3d180bd44dd814c4104cd39c371 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8d0fc3d180bd44dd814c4104cd39c371 2024-11-13T10:38:48,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8d0fc3d180bd44dd814c4104cd39c371, entries=150, sequenceid=79, filesize=11.7 K 2024-11-13T10:38:48,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/59fb59b3606a422f81683387e5af346f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/59fb59b3606a422f81683387e5af346f 2024-11-13T10:38:48,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/59fb59b3606a422f81683387e5af346f, entries=150, sequenceid=79, filesize=11.7 K 2024-11-13T10:38:48,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 99adb18da1f9e813f4728ff7b8100fa3 in 2011ms, sequenceid=79, compaction requested=false 2024-11-13T10:38:48,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:48,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-13T10:38:48,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:48,925 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:38:48,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aa42c83527914cf9ba7db01798ece3f6_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494326837/Put/seqid=0 2024-11-13T10:38:48,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741980_1156 (size=12154) 2024-11-13T10:38:48,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:49,010 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113aa42c83527914cf9ba7db01798ece3f6_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aa42c83527914cf9ba7db01798ece3f6_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:49,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/35aa7828c2ee4c948a9440edec25fcaf, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:49,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/35aa7828c2ee4c948a9440edec25fcaf is 175, key is test_row_0/A:col10/1731494326837/Put/seqid=0 2024-11-13T10:38:49,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:49,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741981_1157 (size=30955) 2024-11-13T10:38:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494389165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494389165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494389219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494389219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494389220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494389327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494389327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494389337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494389337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494389340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,451 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/35aa7828c2ee4c948a9440edec25fcaf 2024-11-13T10:38:49,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/f5cebf2829874cd6ae73a360600276d9 is 50, key is test_row_0/B:col10/1731494326837/Put/seqid=0 2024-11-13T10:38:49,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741982_1158 (size=12001) 2024-11-13T10:38:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494389539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494389539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494389547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494389548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494389557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494389852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494389857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494389859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494389866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494389867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:49,874 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/f5cebf2829874cd6ae73a360600276d9 2024-11-13T10:38:49,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/58e1eb70bdaa420685da8f19e98ace4a is 50, key is test_row_0/C:col10/1731494326837/Put/seqid=0 2024-11-13T10:38:49,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741983_1159 (size=12001) 2024-11-13T10:38:49,918 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/58e1eb70bdaa420685da8f19e98ace4a 2024-11-13T10:38:49,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/35aa7828c2ee4c948a9440edec25fcaf as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/35aa7828c2ee4c948a9440edec25fcaf 2024-11-13T10:38:49,931 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/35aa7828c2ee4c948a9440edec25fcaf, entries=150, sequenceid=93, filesize=30.2 K 2024-11-13T10:38:49,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/f5cebf2829874cd6ae73a360600276d9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f5cebf2829874cd6ae73a360600276d9 2024-11-13T10:38:49,941 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f5cebf2829874cd6ae73a360600276d9, entries=150, sequenceid=93, filesize=11.7 K 2024-11-13T10:38:49,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/58e1eb70bdaa420685da8f19e98ace4a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/58e1eb70bdaa420685da8f19e98ace4a 2024-11-13T10:38:49,965 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/58e1eb70bdaa420685da8f19e98ace4a, entries=150, sequenceid=93, filesize=11.7 K 2024-11-13T10:38:49,967 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 99adb18da1f9e813f4728ff7b8100fa3 in 1041ms, sequenceid=93, compaction requested=true 2024-11-13T10:38:49,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:49,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:49,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-13T10:38:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-13T10:38:49,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-13T10:38:49,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5280 sec 2024-11-13T10:38:49,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.5400 sec 2024-11-13T10:38:50,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:50,371 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:50,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:38:50,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111305476245062e49f59ee3851707c20827_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494330368/Put/seqid=0 2024-11-13T10:38:50,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741984_1160 (size=14594) 2024-11-13T10:38:50,399 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494390392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494390394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494390394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494390392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494390404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,410 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111305476245062e49f59ee3851707c20827_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111305476245062e49f59ee3851707c20827_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:50,412 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/7cf248e0ccf84146a455890b71f8570a, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:50,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/7cf248e0ccf84146a455890b71f8570a is 175, key is test_row_0/A:col10/1731494330368/Put/seqid=0 2024-11-13T10:38:50,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741985_1161 (size=39549) 2024-11-13T10:38:50,431 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/7cf248e0ccf84146a455890b71f8570a 2024-11-13T10:38:50,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/f83966d94d9345cf97bae8b9e66af4c4 is 50, key is test_row_0/B:col10/1731494330368/Put/seqid=0 2024-11-13T10:38:50,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741986_1162 (size=12001) 2024-11-13T10:38:50,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494390508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494390508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494390517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494390517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494390517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-13T10:38:50,601 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:50,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-13T10:38:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T10:38:50,604 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:50,605 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:50,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T10:38:50,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494390717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494390717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494390727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494390727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494390727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:50,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:50,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:50,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:50,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:50,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:50,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:50,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/f83966d94d9345cf97bae8b9e66af4c4 2024-11-13T10:38:50,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/2b9532aa8a1841ec8e3de42e6d1c53c7 is 50, key is test_row_0/C:col10/1731494330368/Put/seqid=0 2024-11-13T10:38:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T10:38:50,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:50,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:50,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:50,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:50,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:50,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:50,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741987_1163 (size=12001) 2024-11-13T10:38:50,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/2b9532aa8a1841ec8e3de42e6d1c53c7 2024-11-13T10:38:50,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/7cf248e0ccf84146a455890b71f8570a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/7cf248e0ccf84146a455890b71f8570a 2024-11-13T10:38:50,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/7cf248e0ccf84146a455890b71f8570a, entries=200, sequenceid=119, filesize=38.6 K 2024-11-13T10:38:51,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/f83966d94d9345cf97bae8b9e66af4c4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f83966d94d9345cf97bae8b9e66af4c4 2024-11-13T10:38:51,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f83966d94d9345cf97bae8b9e66af4c4, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T10:38:51,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/2b9532aa8a1841ec8e3de42e6d1c53c7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b9532aa8a1841ec8e3de42e6d1c53c7 2024-11-13T10:38:51,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b9532aa8a1841ec8e3de42e6d1c53c7, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T10:38:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494391027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 99adb18da1f9e813f4728ff7b8100fa3 in 666ms, sequenceid=119, compaction requested=true 2024-11-13T10:38:51,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:51,038 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:51,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:51,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:51,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:51,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:51,039 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:51,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:51,039 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:51,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:51,040 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/A is initiating minor compaction (all files) 2024-11-13T10:38:51,040 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/A in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,040 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2b4a0ce805e6432eb2520c09ca8e5cd4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ab7e667d0e724d1fa631a86f800f6097, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/35aa7828c2ee4c948a9440edec25fcaf, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/7cf248e0ccf84146a455890b71f8570a] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=146.2 K 2024-11-13T10:38:51,040 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,040 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2b4a0ce805e6432eb2520c09ca8e5cd4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ab7e667d0e724d1fa631a86f800f6097, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/35aa7828c2ee4c948a9440edec25fcaf, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/7cf248e0ccf84146a455890b71f8570a] 2024-11-13T10:38:51,040 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:51,040 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:51,040 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/B is initiating minor compaction (all files) 2024-11-13T10:38:51,040 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/B in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:38:51,040 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1cc2af0f47544cc8830f48cd0f0142e9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8d0fc3d180bd44dd814c4104cd39c371, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f5cebf2829874cd6ae73a360600276d9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f83966d94d9345cf97bae8b9e66af4c4] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=47.0 K 2024-11-13T10:38:51,041 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cc2af0f47544cc8830f48cd0f0142e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494325731 2024-11-13T10:38:51,041 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b4a0ce805e6432eb2520c09ca8e5cd4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494325731 2024-11-13T10:38:51,041 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d0fc3d180bd44dd814c4104cd39c371, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731494326461 2024-11-13T10:38:51,042 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab7e667d0e724d1fa631a86f800f6097, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731494326427 2024-11-13T10:38:51,042 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting f5cebf2829874cd6ae73a360600276d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731494326837 2024-11-13T10:38:51,042 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35aa7828c2ee4c948a9440edec25fcaf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731494326837 2024-11-13T10:38:51,043 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting f83966d94d9345cf97bae8b9e66af4c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494329132 2024-11-13T10:38:51,043 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cf248e0ccf84146a455890b71f8570a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494329132 2024-11-13T10:38:51,080 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:51,081 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/72fa98524732441c8cf2539b92251473 is 50, key is test_row_0/B:col10/1731494330368/Put/seqid=0 2024-11-13T10:38:51,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:51,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:51,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,089 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:51,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111332ec3dc1bdc14675b8a22abecf6a3d30_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494330402/Put/seqid=0 2024-11-13T10:38:51,134 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411136a15af486bc04862ada41c93a2611690_99adb18da1f9e813f4728ff7b8100fa3 store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:51,137 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411136a15af486bc04862ada41c93a2611690_99adb18da1f9e813f4728ff7b8100fa3, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:51,138 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136a15af486bc04862ada41c93a2611690_99adb18da1f9e813f4728ff7b8100fa3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:51,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741989_1165 (size=14644) 2024-11-13T10:38:51,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741988_1164 (size=12241) 2024-11-13T10:38:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494391170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494391172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494391181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494391182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741990_1166 (size=4469) 2024-11-13T10:38:51,205 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#A#compaction#142 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:51,205 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/792c304b638840698da5f5b5d90599ce is 175, key is test_row_0/A:col10/1731494330368/Put/seqid=0 2024-11-13T10:38:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T10:38:51,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741991_1167 (size=31195) 2024-11-13T10:38:51,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494391288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494391288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494391290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494391297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494391515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494391516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494391515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494391516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494391547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:51,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:51,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,573 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:51,582 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111332ec3dc1bdc14675b8a22abecf6a3d30_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111332ec3dc1bdc14675b8a22abecf6a3d30_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:51,582 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/72fa98524732441c8cf2539b92251473 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/72fa98524732441c8cf2539b92251473 2024-11-13T10:38:51,583 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/6c3cb5125d264ee3ad9901a9bad1b2e0, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:51,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/6c3cb5125d264ee3ad9901a9bad1b2e0 is 175, key is test_row_0/A:col10/1731494330402/Put/seqid=0 2024-11-13T10:38:51,597 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/B of 99adb18da1f9e813f4728ff7b8100fa3 into 72fa98524732441c8cf2539b92251473(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:51,597 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:51,597 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/B, priority=12, startTime=1731494331038; duration=0sec 2024-11-13T10:38:51,597 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:51,597 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:B 2024-11-13T10:38:51,597 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:51,600 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:51,601 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/C is initiating minor compaction (all files) 2024-11-13T10:38:51,601 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/C in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,601 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e31606752df141beb86069b9e76f18a8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/59fb59b3606a422f81683387e5af346f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/58e1eb70bdaa420685da8f19e98ace4a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b9532aa8a1841ec8e3de42e6d1c53c7] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=47.0 K 2024-11-13T10:38:51,602 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e31606752df141beb86069b9e76f18a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494325731 2024-11-13T10:38:51,603 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 59fb59b3606a422f81683387e5af346f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731494326461 2024-11-13T10:38:51,603 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e1eb70bdaa420685da8f19e98ace4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731494326837 2024-11-13T10:38:51,604 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9532aa8a1841ec8e3de42e6d1c53c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494329132 2024-11-13T10:38:51,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741992_1168 (size=39599) 2024-11-13T10:38:51,639 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:51,640 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/d9d47381bdbf477e8a21d84677585125 is 50, key is test_row_0/C:col10/1731494330368/Put/seqid=0 2024-11-13T10:38:51,651 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/792c304b638840698da5f5b5d90599ce as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/792c304b638840698da5f5b5d90599ce 2024-11-13T10:38:51,663 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/A of 99adb18da1f9e813f4728ff7b8100fa3 into 792c304b638840698da5f5b5d90599ce(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:51,663 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:51,663 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/A, priority=12, startTime=1731494331037; duration=0sec 2024-11-13T10:38:51,663 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:51,663 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:A 2024-11-13T10:38:51,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741993_1169 (size=12241) 2024-11-13T10:38:51,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:51,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:51,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T10:38:51,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494391829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494391841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494391841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494391842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:51,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:51,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:51,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:51,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,019 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/6c3cb5125d264ee3ad9901a9bad1b2e0 2024-11-13T10:38:52,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:52,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/7d0681a106e24ef2984ee78b695e961b is 50, key is test_row_0/B:col10/1731494330402/Put/seqid=0 2024-11-13T10:38:52,089 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/d9d47381bdbf477e8a21d84677585125 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d9d47381bdbf477e8a21d84677585125 2024-11-13T10:38:52,096 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/C of 99adb18da1f9e813f4728ff7b8100fa3 into d9d47381bdbf477e8a21d84677585125(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:52,096 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:52,096 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/C, priority=12, startTime=1731494331039; duration=0sec 2024-11-13T10:38:52,097 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:52,097 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:C 2024-11-13T10:38:52,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741994_1170 (size=12051) 2024-11-13T10:38:52,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:52,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494392336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:52,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494392357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:52,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494392357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:52,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494392357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:52,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:52,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:52,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/7d0681a106e24ef2984ee78b695e961b 2024-11-13T10:38:52,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:52,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/78327cf9924b4a47b075c3c6d8150242 is 50, key is test_row_0/C:col10/1731494330402/Put/seqid=0 2024-11-13T10:38:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:52,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741995_1171 (size=12051) 2024-11-13T10:38:52,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/78327cf9924b4a47b075c3c6d8150242 2024-11-13T10:38:52,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/6c3cb5125d264ee3ad9901a9bad1b2e0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6c3cb5125d264ee3ad9901a9bad1b2e0 2024-11-13T10:38:52,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494392567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:52,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6c3cb5125d264ee3ad9901a9bad1b2e0, entries=200, sequenceid=130, filesize=38.7 K 2024-11-13T10:38:52,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/7d0681a106e24ef2984ee78b695e961b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/7d0681a106e24ef2984ee78b695e961b 2024-11-13T10:38:52,573 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_2' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:38:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:52,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:52,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/7d0681a106e24ef2984ee78b695e961b, entries=150, sequenceid=130, filesize=11.8 K 2024-11-13T10:38:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/78327cf9924b4a47b075c3c6d8150242 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/78327cf9924b4a47b075c3c6d8150242 2024-11-13T10:38:52,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/78327cf9924b4a47b075c3c6d8150242, entries=150, sequenceid=130, filesize=11.8 K 2024-11-13T10:38:52,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 99adb18da1f9e813f4728ff7b8100fa3 in 1553ms, sequenceid=130, compaction requested=false 2024-11-13T10:38:52,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:52,624 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T10:38:52,624 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T10:38:52,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-13T10:38:52,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:52,682 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:38:52,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c5a43ef7bb594e3bb2758cef8e9046ba_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494331180/Put/seqid=0 2024-11-13T10:38:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T10:38:52,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741996_1172 (size=12304) 2024-11-13T10:38:53,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,168 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c5a43ef7bb594e3bb2758cef8e9046ba_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c5a43ef7bb594e3bb2758cef8e9046ba_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:53,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/e6f96b9636e044eca6beb67741846b6e, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:53,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/e6f96b9636e044eca6beb67741846b6e is 175, key is test_row_0/A:col10/1731494331180/Put/seqid=0 2024-11-13T10:38:53,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741997_1173 (size=31105) 2024-11-13T10:38:53,177 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/e6f96b9636e044eca6beb67741846b6e 2024-11-13T10:38:53,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/87f1c05febd9491fa1c37d201439ee1a is 50, key is test_row_0/B:col10/1731494331180/Put/seqid=0 2024-11-13T10:38:53,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741998_1174 (size=12151) 2024-11-13T10:38:53,205 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/87f1c05febd9491fa1c37d201439ee1a 2024-11-13T10:38:53,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/e6679fe0cfef4513b1c61f413f6f27c2 is 50, key is test_row_0/C:col10/1731494331180/Put/seqid=0 2024-11-13T10:38:53,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741999_1175 (size=12151) 2024-11-13T10:38:53,270 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/e6679fe0cfef4513b1c61f413f6f27c2 2024-11-13T10:38:53,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/e6f96b9636e044eca6beb67741846b6e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/e6f96b9636e044eca6beb67741846b6e 2024-11-13T10:38:53,289 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/e6f96b9636e044eca6beb67741846b6e, entries=150, sequenceid=159, filesize=30.4 K 2024-11-13T10:38:53,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/87f1c05febd9491fa1c37d201439ee1a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/87f1c05febd9491fa1c37d201439ee1a 2024-11-13T10:38:53,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,297 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/87f1c05febd9491fa1c37d201439ee1a, entries=150, sequenceid=159, filesize=11.9 K 2024-11-13T10:38:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/e6679fe0cfef4513b1c61f413f6f27c2 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e6679fe0cfef4513b1c61f413f6f27c2 2024-11-13T10:38:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,325 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e6679fe0cfef4513b1c61f413f6f27c2, entries=150, sequenceid=159, filesize=11.9 K 2024-11-13T10:38:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,327 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 99adb18da1f9e813f4728ff7b8100fa3 in 644ms, sequenceid=159, compaction requested=true 2024-11-13T10:38:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:53,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:53,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-13T10:38:53,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-13T10:38:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-13T10:38:53,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7250 sec 2024-11-13T10:38:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.7320 sec 2024-11-13T10:38:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:53,429 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:38:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f235afb74bd143f89872a8921f048f1f_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494333417/Put/seqid=0 2024-11-13T10:38:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742001_1177 (size=24758) 2024-11-13T10:38:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,524 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f235afb74bd143f89872a8921f048f1f_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f235afb74bd143f89872a8921f048f1f_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/bbf3f01193634d918e15ad17c38077a1, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/bbf3f01193634d918e15ad17c38077a1 is 175, key is test_row_0/A:col10/1731494333417/Put/seqid=0 2024-11-13T10:38:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742000_1176 (size=74394) 2024-11-13T10:38:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494393556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494393560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494393563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494393573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:53,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494393679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494393679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494393687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494393687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494393897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494393897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494393897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494393897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:53,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:53,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/bbf3f01193634d918e15ad17c38077a1 2024-11-13T10:38:53,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/11b2dfc5a296473cb6c77c7c59a529bf is 50, key is test_row_0/B:col10/1731494333417/Put/seqid=0 2024-11-13T10:38:53,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742002_1178 (size=12151) 2024-11-13T10:38:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494394207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494394207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:54,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494394220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:54,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:54,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494394221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:54,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/11b2dfc5a296473cb6c77c7c59a529bf 2024-11-13T10:38:54,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/931dc67b185342f980f7bef35fede76e is 50, key is test_row_0/C:col10/1731494333417/Put/seqid=0 2024-11-13T10:38:54,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742003_1179 (size=12151) 2024-11-13T10:38:54,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494394586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:54,589 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_2' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:38:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:54,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494394716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:54,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494394727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:54,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494394739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:54,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494394739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-13T10:38:54,767 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:54,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-13T10:38:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T10:38:54,770 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:54,771 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:54,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:54,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/931dc67b185342f980f7bef35fede76e 2024-11-13T10:38:54,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/bbf3f01193634d918e15ad17c38077a1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/bbf3f01193634d918e15ad17c38077a1 2024-11-13T10:38:54,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/bbf3f01193634d918e15ad17c38077a1, entries=400, sequenceid=170, filesize=72.7 K 2024-11-13T10:38:54,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/11b2dfc5a296473cb6c77c7c59a529bf as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/11b2dfc5a296473cb6c77c7c59a529bf 2024-11-13T10:38:54,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/11b2dfc5a296473cb6c77c7c59a529bf, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T10:38:54,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/931dc67b185342f980f7bef35fede76e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/931dc67b185342f980f7bef35fede76e 2024-11-13T10:38:54,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/931dc67b185342f980f7bef35fede76e, entries=150, sequenceid=170, filesize=11.9 K 2024-11-13T10:38:54,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 99adb18da1f9e813f4728ff7b8100fa3 in 1430ms, sequenceid=170, compaction requested=true 2024-11-13T10:38:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:54,859 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:54,859 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:54,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:54,862 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176293 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:54,862 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/A is initiating minor compaction (all files) 2024-11-13T10:38:54,862 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:54,862 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/A in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:54,862 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/B is initiating minor compaction (all files) 2024-11-13T10:38:54,862 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/B in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:54,862 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/792c304b638840698da5f5b5d90599ce, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6c3cb5125d264ee3ad9901a9bad1b2e0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/e6f96b9636e044eca6beb67741846b6e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/bbf3f01193634d918e15ad17c38077a1] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=172.2 K 2024-11-13T10:38:54,862 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/72fa98524732441c8cf2539b92251473, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/7d0681a106e24ef2984ee78b695e961b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/87f1c05febd9491fa1c37d201439ee1a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/11b2dfc5a296473cb6c77c7c59a529bf] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=47.5 K 2024-11-13T10:38:54,862 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:54,863 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/792c304b638840698da5f5b5d90599ce, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6c3cb5125d264ee3ad9901a9bad1b2e0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/e6f96b9636e044eca6beb67741846b6e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/bbf3f01193634d918e15ad17c38077a1] 2024-11-13T10:38:54,863 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 792c304b638840698da5f5b5d90599ce, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494329132 2024-11-13T10:38:54,863 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 72fa98524732441c8cf2539b92251473, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494329132 2024-11-13T10:38:54,866 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c3cb5125d264ee3ad9901a9bad1b2e0, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494330381 2024-11-13T10:38:54,866 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d0681a106e24ef2984ee78b695e961b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494330391 2024-11-13T10:38:54,867 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f96b9636e044eca6beb67741846b6e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731494331168 2024-11-13T10:38:54,867 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 87f1c05febd9491fa1c37d201439ee1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731494331168 2024-11-13T10:38:54,868 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf3f01193634d918e15ad17c38077a1, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731494333389 2024-11-13T10:38:54,868 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b2dfc5a296473cb6c77c7c59a529bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731494333389 2024-11-13T10:38:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T10:38:54,889 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:54,894 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#B#compaction#154 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:54,894 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/b79325a6cfb4472492eff1e92efedc1f is 50, key is test_row_0/B:col10/1731494333417/Put/seqid=0 2024-11-13T10:38:54,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T10:38:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:54,928 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:38:54,932 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411133c3726ad41184cffa63731415a321c1a_99adb18da1f9e813f4728ff7b8100fa3 store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:54,945 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411133c3726ad41184cffa63731415a321c1a_99adb18da1f9e813f4728ff7b8100fa3, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:54,945 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133c3726ad41184cffa63731415a321c1a_99adb18da1f9e813f4728ff7b8100fa3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:54,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742004_1180 (size=12527) 2024-11-13T10:38:54,976 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/b79325a6cfb4472492eff1e92efedc1f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b79325a6cfb4472492eff1e92efedc1f 2024-11-13T10:38:55,011 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/B of 99adb18da1f9e813f4728ff7b8100fa3 into b79325a6cfb4472492eff1e92efedc1f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:55,011 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:55,011 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/B, priority=12, startTime=1731494334859; duration=0sec 2024-11-13T10:38:55,011 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:55,011 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:B 2024-11-13T10:38:55,012 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:38:55,017 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:38:55,017 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/C is initiating minor compaction (all files) 2024-11-13T10:38:55,017 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/C in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:55,017 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d9d47381bdbf477e8a21d84677585125, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/78327cf9924b4a47b075c3c6d8150242, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e6679fe0cfef4513b1c61f413f6f27c2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/931dc67b185342f980f7bef35fede76e] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=47.5 K 2024-11-13T10:38:55,018 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting d9d47381bdbf477e8a21d84677585125, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494329132 2024-11-13T10:38:55,018 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 78327cf9924b4a47b075c3c6d8150242, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494330391 2024-11-13T10:38:55,020 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e6679fe0cfef4513b1c61f413f6f27c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731494331168 2024-11-13T10:38:55,025 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 931dc67b185342f980f7bef35fede76e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731494333389 2024-11-13T10:38:55,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c2fcbab188da4064855c7dc280b3e038_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494333567/Put/seqid=0 2024-11-13T10:38:55,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742005_1181 (size=4469) 2024-11-13T10:38:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T10:38:55,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742006_1182 (size=12304) 2024-11-13T10:38:55,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,093 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:55,094 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/7eb78e4936a9481f9f6c0e57564bdc05 is 50, key is test_row_0/C:col10/1731494333417/Put/seqid=0 2024-11-13T10:38:55,095 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c2fcbab188da4064855c7dc280b3e038_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c2fcbab188da4064855c7dc280b3e038_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:55,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/d567d4080e904760b4ca4611b7fbf486, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:55,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/d567d4080e904760b4ca4611b7fbf486 is 175, key is test_row_0/A:col10/1731494333567/Put/seqid=0 2024-11-13T10:38:55,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742007_1183 (size=31105) 2024-11-13T10:38:55,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742008_1184 (size=12527) 2024-11-13T10:38:55,109 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/d567d4080e904760b4ca4611b7fbf486 2024-11-13T10:38:55,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/b56c475b21ea4ade8bdda47eb392a72a is 50, key is test_row_0/B:col10/1731494333567/Put/seqid=0 2024-11-13T10:38:55,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742009_1185 (size=12151) 2024-11-13T10:38:55,163 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/b56c475b21ea4ade8bdda47eb392a72a 2024-11-13T10:38:55,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/26dfd1d9db4d4cf089d65f3db85a4f9c is 50, key is test_row_0/C:col10/1731494333567/Put/seqid=0 2024-11-13T10:38:55,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742010_1186 (size=12151) 2024-11-13T10:38:55,198 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/26dfd1d9db4d4cf089d65f3db85a4f9c 2024-11-13T10:38:55,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/d567d4080e904760b4ca4611b7fbf486 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/d567d4080e904760b4ca4611b7fbf486 2024-11-13T10:38:55,210 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/d567d4080e904760b4ca4611b7fbf486, entries=150, sequenceid=195, filesize=30.4 K 2024-11-13T10:38:55,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/b56c475b21ea4ade8bdda47eb392a72a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b56c475b21ea4ade8bdda47eb392a72a 2024-11-13T10:38:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,217 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b56c475b21ea4ade8bdda47eb392a72a, entries=150, sequenceid=195, filesize=11.9 K 2024-11-13T10:38:55,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/26dfd1d9db4d4cf089d65f3db85a4f9c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/26dfd1d9db4d4cf089d65f3db85a4f9c 2024-11-13T10:38:55,224 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/26dfd1d9db4d4cf089d65f3db85a4f9c, entries=150, sequenceid=195, filesize=11.9 K 2024-11-13T10:38:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,227 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 99adb18da1f9e813f4728ff7b8100fa3 in 299ms, sequenceid=195, compaction requested=false 2024-11-13T10:38:55,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:55,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:55,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-13T10:38:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-13T10:38:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-13T10:38:55,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 461 msec 2024-11-13T10:38:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 476 msec 2024-11-13T10:38:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T10:38:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,398 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-13T10:38:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T10:38:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,401 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,402 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:55,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,449 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#A#compaction#153 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,450 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/a389d6bcab3b4d33b744a4e7cffcbd56 is 175, key is test_row_0/A:col10/1731494333417/Put/seqid=0 2024-11-13T10:38:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742011_1187 (size=31481) 2024-11-13T10:38:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,492 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/a389d6bcab3b4d33b744a4e7cffcbd56 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a389d6bcab3b4d33b744a4e7cffcbd56 2024-11-13T10:38:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,502 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/A of 99adb18da1f9e813f4728ff7b8100fa3 into a389d6bcab3b4d33b744a4e7cffcbd56(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:55,503 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:55,503 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/A, priority=12, startTime=1731494334859; duration=0sec 2024-11-13T10:38:55,503 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:55,503 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:A 2024-11-13T10:38:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T10:38:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,532 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/7eb78e4936a9481f9f6c0e57564bdc05 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/7eb78e4936a9481f9f6c0e57564bdc05 2024-11-13T10:38:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,545 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/C of 99adb18da1f9e813f4728ff7b8100fa3 into 7eb78e4936a9481f9f6c0e57564bdc05(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:55,545 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,545 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/C, priority=12, startTime=1731494334859; duration=0sec 2024-11-13T10:38:55,545 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:55,545 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:C 2024-11-13T10:38:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T10:38:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:55,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:55,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-13T10:38:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-13T10:38:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-13T10:38:55,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 163 msec 2024-11-13T10:38:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 170 msec 2024-11-13T10:38:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T10:38:55,717 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-13T10:38:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T10:38:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,739 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,749 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:55,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T10:38:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,889 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:55,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:38:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111308035d01cd7f4dd68d9abd5d70278bcf_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494335875/Put/seqid=0 2024-11-13T10:38:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T10:38:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:55,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:55,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:55,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742012_1188 (size=24758) 2024-11-13T10:38:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:55,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494395964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:55,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494395964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:55,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494395972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:55,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494395968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:55,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T10:38:56,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T10:38:56,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:56,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494396077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494396077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494396086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494396097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T10:38:56,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:56,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494396287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494396287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494396296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494396307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,349 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:56,355 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111308035d01cd7f4dd68d9abd5d70278bcf_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308035d01cd7f4dd68d9abd5d70278bcf_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:56,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ffd6abc6abf34481bc24453af2e14f02, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:56,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ffd6abc6abf34481bc24453af2e14f02 is 175, key is test_row_0/A:col10/1731494335875/Put/seqid=0 2024-11-13T10:38:56,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742013_1189 (size=74394) 2024-11-13T10:38:56,361 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ffd6abc6abf34481bc24453af2e14f02 2024-11-13T10:38:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T10:38:56,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/24e87b781840497caf7d0aa70f295e12 is 50, key is test_row_0/B:col10/1731494335875/Put/seqid=0 2024-11-13T10:38:56,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742014_1190 (size=12151) 2024-11-13T10:38:56,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/24e87b781840497caf7d0aa70f295e12 2024-11-13T10:38:56,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T10:38:56,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:56,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/03e4915e7506469586415cd74b831f68 is 50, key is test_row_0/C:col10/1731494335875/Put/seqid=0 2024-11-13T10:38:56,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742015_1191 (size=12151) 2024-11-13T10:38:56,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T10:38:56,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:56,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494396597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494396597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494396606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494396616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:56,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T10:38:56,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:56,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:56,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/03e4915e7506469586415cd74b831f68 2024-11-13T10:38:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ffd6abc6abf34481bc24453af2e14f02 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ffd6abc6abf34481bc24453af2e14f02 2024-11-13T10:38:56,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ffd6abc6abf34481bc24453af2e14f02, entries=400, sequenceid=210, filesize=72.7 K 2024-11-13T10:38:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/24e87b781840497caf7d0aa70f295e12 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/24e87b781840497caf7d0aa70f295e12 2024-11-13T10:38:56,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/24e87b781840497caf7d0aa70f295e12, entries=150, sequenceid=210, filesize=11.9 K 2024-11-13T10:38:56,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/03e4915e7506469586415cd74b831f68 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/03e4915e7506469586415cd74b831f68 2024-11-13T10:38:56,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/03e4915e7506469586415cd74b831f68, entries=150, sequenceid=210, filesize=11.9 K 2024-11-13T10:38:56,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 99adb18da1f9e813f4728ff7b8100fa3 in 941ms, sequenceid=210, compaction requested=true 2024-11-13T10:38:56,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:56,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:56,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:56,831 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:56,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:56,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:56,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:56,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:56,831 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:56,833 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136980 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:56,833 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/A is initiating minor compaction (all files) 2024-11-13T10:38:56,833 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/A in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,833 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a389d6bcab3b4d33b744a4e7cffcbd56, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/d567d4080e904760b4ca4611b7fbf486, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ffd6abc6abf34481bc24453af2e14f02] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=133.8 K 2024-11-13T10:38:56,833 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,833 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a389d6bcab3b4d33b744a4e7cffcbd56, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/d567d4080e904760b4ca4611b7fbf486, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ffd6abc6abf34481bc24453af2e14f02] 2024-11-13T10:38:56,834 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:56,834 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/B is initiating minor compaction (all files) 2024-11-13T10:38:56,834 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/B in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,834 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b79325a6cfb4472492eff1e92efedc1f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b56c475b21ea4ade8bdda47eb392a72a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/24e87b781840497caf7d0aa70f295e12] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=36.0 K 2024-11-13T10:38:56,834 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a389d6bcab3b4d33b744a4e7cffcbd56, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731494333389 2024-11-13T10:38:56,834 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting b79325a6cfb4472492eff1e92efedc1f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731494333389 2024-11-13T10:38:56,834 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d567d4080e904760b4ca4611b7fbf486, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731494333542 2024-11-13T10:38:56,835 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting b56c475b21ea4ade8bdda47eb392a72a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731494333542 2024-11-13T10:38:56,835 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd6abc6abf34481bc24453af2e14f02, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731494335814 2024-11-13T10:38:56,835 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e87b781840497caf7d0aa70f295e12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731494335867 2024-11-13T10:38:56,849 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:56,850 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/499dab5194a44cf7af07461bba1d0785 is 50, key is test_row_0/B:col10/1731494335875/Put/seqid=0 2024-11-13T10:38:56,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T10:38:56,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,852 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:38:56,857 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:56,866 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113512fd492c0594d2698c620e0b817830b_99adb18da1f9e813f4728ff7b8100fa3 store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:56,868 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113512fd492c0594d2698c620e0b817830b_99adb18da1f9e813f4728ff7b8100fa3, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:56,869 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113512fd492c0594d2698c620e0b817830b_99adb18da1f9e813f4728ff7b8100fa3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T10:38:56,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742016_1192 (size=12629) 2024-11-13T10:38:56,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111376934e01d58e4b119e07a4c308bd7422_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494335970/Put/seqid=0 2024-11-13T10:38:56,887 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/499dab5194a44cf7af07461bba1d0785 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/499dab5194a44cf7af07461bba1d0785 2024-11-13T10:38:56,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742017_1193 (size=4469) 2024-11-13T10:38:56,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742018_1194 (size=12304) 2024-11-13T10:38:56,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:56,893 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#A#compaction#163 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:56,894 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/3008f4de6bb24218943adc347ccff761 is 175, key is test_row_0/A:col10/1731494335875/Put/seqid=0 2024-11-13T10:38:56,897 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/B of 99adb18da1f9e813f4728ff7b8100fa3 into 499dab5194a44cf7af07461bba1d0785(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:56,897 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:56,897 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/B, priority=13, startTime=1731494336831; duration=0sec 2024-11-13T10:38:56,897 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:56,897 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:B 2024-11-13T10:38:56,897 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:56,899 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111376934e01d58e4b119e07a4c308bd7422_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111376934e01d58e4b119e07a4c308bd7422_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:56,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/95ee2aa041774f8daabc47b201e38805, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:56,908 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/95ee2aa041774f8daabc47b201e38805 is 175, key is test_row_0/A:col10/1731494335970/Put/seqid=0 2024-11-13T10:38:56,908 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/C is initiating minor compaction (all files) 2024-11-13T10:38:56,908 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/C in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:56,908 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/7eb78e4936a9481f9f6c0e57564bdc05, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/26dfd1d9db4d4cf089d65f3db85a4f9c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/03e4915e7506469586415cd74b831f68] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=36.0 K 2024-11-13T10:38:56,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742019_1195 (size=31583) 2024-11-13T10:38:56,908 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eb78e4936a9481f9f6c0e57564bdc05, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731494333389 2024-11-13T10:38:56,909 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 26dfd1d9db4d4cf089d65f3db85a4f9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731494333542 2024-11-13T10:38:56,910 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 03e4915e7506469586415cd74b831f68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731494335867 2024-11-13T10:38:56,926 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:56,927 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/80ddb6df09ac4ed081f9324007c75ae9 is 50, key is test_row_0/C:col10/1731494335875/Put/seqid=0 2024-11-13T10:38:56,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742020_1196 (size=31105) 2024-11-13T10:38:56,944 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/95ee2aa041774f8daabc47b201e38805 2024-11-13T10:38:56,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742021_1197 (size=12629) 2024-11-13T10:38:56,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/64875e6e959445d096ea415f95d91871 is 50, key is test_row_0/B:col10/1731494335970/Put/seqid=0 2024-11-13T10:38:56,965 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/80ddb6df09ac4ed081f9324007c75ae9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/80ddb6df09ac4ed081f9324007c75ae9 2024-11-13T10:38:56,971 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/C of 99adb18da1f9e813f4728ff7b8100fa3 into 80ddb6df09ac4ed081f9324007c75ae9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:56,971 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:56,971 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/C, priority=13, startTime=1731494336831; duration=0sec 2024-11-13T10:38:56,972 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:56,972 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:C 2024-11-13T10:38:56,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742022_1198 (size=12151) 2024-11-13T10:38:56,976 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/64875e6e959445d096ea415f95d91871 2024-11-13T10:38:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/bf8d54812ce24d55a28b7d2e39f62022 is 50, key is test_row_0/C:col10/1731494335970/Put/seqid=0 2024-11-13T10:38:56,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742023_1199 (size=12151) 2024-11-13T10:38:56,999 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/bf8d54812ce24d55a28b7d2e39f62022 2024-11-13T10:38:57,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/95ee2aa041774f8daabc47b201e38805 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/95ee2aa041774f8daabc47b201e38805 2024-11-13T10:38:57,018 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/95ee2aa041774f8daabc47b201e38805, entries=150, sequenceid=235, filesize=30.4 K 2024-11-13T10:38:57,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/64875e6e959445d096ea415f95d91871 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/64875e6e959445d096ea415f95d91871 2024-11-13T10:38:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,028 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/64875e6e959445d096ea415f95d91871, entries=150, sequenceid=235, filesize=11.9 K 2024-11-13T10:38:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/bf8d54812ce24d55a28b7d2e39f62022 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/bf8d54812ce24d55a28b7d2e39f62022 2024-11-13T10:38:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,036 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/bf8d54812ce24d55a28b7d2e39f62022, entries=150, sequenceid=235, filesize=11.9 K 2024-11-13T10:38:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,037 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 99adb18da1f9e813f4728ff7b8100fa3 in 185ms, sequenceid=235, compaction requested=false 2024-11-13T10:38:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:57,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-13T10:38:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-13T10:38:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T10:38:57,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2900 sec 2024-11-13T10:38:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.3250 sec 2024-11-13T10:38:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:57,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:38:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c69ad77a935642ee9c355ce389d91134_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494337196/Put/seqid=0 2024-11-13T10:38:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742024_1200 (size=19774) 2024-11-13T10:38:57,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494397244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494397246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494397248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494397251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,314 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/3008f4de6bb24218943adc347ccff761 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/3008f4de6bb24218943adc347ccff761 2024-11-13T10:38:57,321 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/A of 99adb18da1f9e813f4728ff7b8100fa3 into 3008f4de6bb24218943adc347ccff761(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:57,321 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:57,321 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/A, priority=13, startTime=1731494336831; duration=0sec 2024-11-13T10:38:57,321 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:57,321 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:A 2024-11-13T10:38:57,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494397357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494397357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494397357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494397367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,405 INFO [master/770665a7984d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T10:38:57,405 INFO [master/770665a7984d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T10:38:57,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494397566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494397567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494397567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494397577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,640 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:57,645 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c69ad77a935642ee9c355ce389d91134_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c69ad77a935642ee9c355ce389d91134_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:57,646 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/caae05da88104d10adf4b94895f1bf4b, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:57,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/caae05da88104d10adf4b94895f1bf4b is 175, key is test_row_0/A:col10/1731494337196/Put/seqid=0 2024-11-13T10:38:57,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742025_1201 (size=57033) 2024-11-13T10:38:57,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494397876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494397876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494397877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T10:38:57,887 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:38:57,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:38:57,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494397887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-13T10:38:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T10:38:57,889 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:38:57,890 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:38:57,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:38:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T10:38:58,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:58,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:58,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,053 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/caae05da88104d10adf4b94895f1bf4b 2024-11-13T10:38:58,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/34c4e87f67a14ec880e270d2cd064287 is 50, key is test_row_0/B:col10/1731494337196/Put/seqid=0 2024-11-13T10:38:58,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742026_1202 (size=12151) 2024-11-13T10:38:58,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T10:38:58,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:58,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:58,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494398387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494398387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494398388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:58,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494398397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:58,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/34c4e87f67a14ec880e270d2cd064287 2024-11-13T10:38:58,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/4ea3363b796c486a848343e37581e5a8 is 50, key is test_row_0/C:col10/1731494337196/Put/seqid=0 2024-11-13T10:38:58,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742027_1203 (size=12151) 2024-11-13T10:38:58,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/4ea3363b796c486a848343e37581e5a8 2024-11-13T10:38:58,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/caae05da88104d10adf4b94895f1bf4b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/caae05da88104d10adf4b94895f1bf4b 2024-11-13T10:38:58,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:58,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:58,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/caae05da88104d10adf4b94895f1bf4b, entries=300, sequenceid=248, filesize=55.7 K 2024-11-13T10:38:58,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/34c4e87f67a14ec880e270d2cd064287 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/34c4e87f67a14ec880e270d2cd064287 2024-11-13T10:38:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/34c4e87f67a14ec880e270d2cd064287, entries=150, sequenceid=248, filesize=11.9 K 2024-11-13T10:38:58,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/4ea3363b796c486a848343e37581e5a8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/4ea3363b796c486a848343e37581e5a8 2024-11-13T10:38:58,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/4ea3363b796c486a848343e37581e5a8, entries=150, sequenceid=248, filesize=11.9 K 2024-11-13T10:38:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T10:38:58,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 99adb18da1f9e813f4728ff7b8100fa3 in 1319ms, sequenceid=248, compaction requested=true 2024-11-13T10:38:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:58,518 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:38:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:58,518 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:58,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:38:58,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:58,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:38:58,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:58,520 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:58,520 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/A is initiating minor compaction (all files) 2024-11-13T10:38:58,520 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/A in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,520 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/3008f4de6bb24218943adc347ccff761, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/95ee2aa041774f8daabc47b201e38805, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/caae05da88104d10adf4b94895f1bf4b] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=116.9 K 2024-11-13T10:38:58,520 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,520 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/3008f4de6bb24218943adc347ccff761, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/95ee2aa041774f8daabc47b201e38805, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/caae05da88104d10adf4b94895f1bf4b] 2024-11-13T10:38:58,521 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:58,521 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/B is initiating minor compaction (all files) 2024-11-13T10:38:58,521 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/B in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,521 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3008f4de6bb24218943adc347ccff761, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731494335867 2024-11-13T10:38:58,521 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/499dab5194a44cf7af07461bba1d0785, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/64875e6e959445d096ea415f95d91871, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/34c4e87f67a14ec880e270d2cd064287] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=36.1 K 2024-11-13T10:38:58,521 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95ee2aa041774f8daabc47b201e38805, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731494335961 2024-11-13T10:38:58,521 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 499dab5194a44cf7af07461bba1d0785, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731494335867 2024-11-13T10:38:58,522 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting caae05da88104d10adf4b94895f1bf4b, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731494337165 2024-11-13T10:38:58,522 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 64875e6e959445d096ea415f95d91871, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731494335961 2024-11-13T10:38:58,522 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 34c4e87f67a14ec880e270d2cd064287, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731494337194 2024-11-13T10:38:58,531 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:58,536 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:58,537 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/cc63be313dce4a17ba9e236f8f5b15ed is 50, key is test_row_0/B:col10/1731494337196/Put/seqid=0 2024-11-13T10:38:58,537 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113342c7f27bd9a42c882c5cc3e793b5973_99adb18da1f9e813f4728ff7b8100fa3 store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:58,538 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113342c7f27bd9a42c882c5cc3e793b5973_99adb18da1f9e813f4728ff7b8100fa3, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:58,538 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113342c7f27bd9a42c882c5cc3e793b5973_99adb18da1f9e813f4728ff7b8100fa3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:58,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742029_1205 (size=12731) 2024-11-13T10:38:58,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742028_1204 (size=4469) 2024-11-13T10:38:58,555 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#A#compaction#171 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:58,555 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/c5c79fe358cf4b13a62c759d9da142a8 is 175, key is test_row_0/A:col10/1731494337196/Put/seqid=0 2024-11-13T10:38:58,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742030_1206 (size=31685) 2024-11-13T10:38:58,568 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/c5c79fe358cf4b13a62c759d9da142a8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/c5c79fe358cf4b13a62c759d9da142a8 2024-11-13T10:38:58,574 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/A of 99adb18da1f9e813f4728ff7b8100fa3 into c5c79fe358cf4b13a62c759d9da142a8(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:58,574 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:58,575 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/A, priority=13, startTime=1731494338518; duration=0sec 2024-11-13T10:38:58,575 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:38:58,575 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:A 2024-11-13T10:38:58,575 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:38:58,576 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:38:58,576 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/C is initiating minor compaction (all files) 2024-11-13T10:38:58,576 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/C in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,576 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/80ddb6df09ac4ed081f9324007c75ae9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/bf8d54812ce24d55a28b7d2e39f62022, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/4ea3363b796c486a848343e37581e5a8] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=36.1 K 2024-11-13T10:38:58,577 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80ddb6df09ac4ed081f9324007c75ae9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731494335867 2024-11-13T10:38:58,578 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf8d54812ce24d55a28b7d2e39f62022, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731494335961 2024-11-13T10:38:58,578 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea3363b796c486a848343e37581e5a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731494337194 2024-11-13T10:38:58,587 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#C#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:38:58,587 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/eb826b923fa540e7900e680e02ea4ee5 is 50, key is test_row_0/C:col10/1731494337196/Put/seqid=0 2024-11-13T10:38:58,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742031_1207 (size=12731) 2024-11-13T10:38:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:58,619 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:38:58,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:38:58,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111340201947582a4b9485c9f44e16ea69ed_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494337245/Put/seqid=0 2024-11-13T10:38:58,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742032_1208 (size=14994) 2024-11-13T10:38:58,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494398644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:58,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494398757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:58,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:58,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:58,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:58,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,959 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/cc63be313dce4a17ba9e236f8f5b15ed as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/cc63be313dce4a17ba9e236f8f5b15ed 2024-11-13T10:38:58,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:58,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:58,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:58,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:58,967 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/B of 99adb18da1f9e813f4728ff7b8100fa3 into cc63be313dce4a17ba9e236f8f5b15ed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:58,968 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:58,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:58,968 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/B, priority=13, startTime=1731494338518; duration=0sec 2024-11-13T10:38:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494398967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:58,968 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:58,968 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:B 2024-11-13T10:38:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:58,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:59,011 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/eb826b923fa540e7900e680e02ea4ee5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/eb826b923fa540e7900e680e02ea4ee5 2024-11-13T10:38:59,018 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/C of 99adb18da1f9e813f4728ff7b8100fa3 into eb826b923fa540e7900e680e02ea4ee5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:38:59,018 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:59,018 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/C, priority=13, startTime=1731494338519; duration=0sec 2024-11-13T10:38:59,018 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:38:59,018 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:C 2024-11-13T10:38:59,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T10:38:59,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:59,039 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111340201947582a4b9485c9f44e16ea69ed_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111340201947582a4b9485c9f44e16ea69ed_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:59,040 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ff27708ec77f41cea3aadbddb814f2c6, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:59,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ff27708ec77f41cea3aadbddb814f2c6 is 175, key is test_row_0/A:col10/1731494337245/Put/seqid=0 2024-11-13T10:38:59,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742033_1209 (size=39949) 2024-11-13T10:38:59,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:59,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:59,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494399277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:59,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:59,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:59,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:59,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:59,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:59,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:59,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494399400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:59,404 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:38:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:59,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494399407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:59,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494399407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:59,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494399407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:59,410 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:38:59,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:59,412 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:38:59,413 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:38:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:38:59,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:59,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:38:59,447 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ff27708ec77f41cea3aadbddb814f2c6 2024-11-13T10:38:59,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/e8f61e84dd034374bbdbfb84ea77e27e is 50, key is test_row_0/B:col10/1731494337245/Put/seqid=0 2024-11-13T10:38:59,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742034_1210 (size=12301) 2024-11-13T10:38:59,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/e8f61e84dd034374bbdbfb84ea77e27e 2024-11-13T10:38:59,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/459338a211b043a2b1270ad4f15ab6b9 is 50, key is test_row_0/C:col10/1731494337245/Put/seqid=0 2024-11-13T10:38:59,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742035_1211 (size=12301) 2024-11-13T10:38:59,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/459338a211b043a2b1270ad4f15ab6b9 2024-11-13T10:38:59,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/ff27708ec77f41cea3aadbddb814f2c6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ff27708ec77f41cea3aadbddb814f2c6 2024-11-13T10:38:59,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ff27708ec77f41cea3aadbddb814f2c6, entries=200, sequenceid=276, filesize=39.0 K 2024-11-13T10:38:59,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/e8f61e84dd034374bbdbfb84ea77e27e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/e8f61e84dd034374bbdbfb84ea77e27e 2024-11-13T10:38:59,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/e8f61e84dd034374bbdbfb84ea77e27e, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T10:38:59,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/459338a211b043a2b1270ad4f15ab6b9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/459338a211b043a2b1270ad4f15ab6b9 2024-11-13T10:38:59,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/459338a211b043a2b1270ad4f15ab6b9, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T10:38:59,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 99adb18da1f9e813f4728ff7b8100fa3 in 911ms, sequenceid=276, compaction requested=false 2024-11-13T10:38:59,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:38:59,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T10:38:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:38:59,596 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T10:38:59,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111332628510e576496ab56180c13f1648e7_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494338642/Put/seqid=0 2024-11-13T10:38:59,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742036_1212 (size=12454) 2024-11-13T10:38:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:38:59,697 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111332628510e576496ab56180c13f1648e7_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111332628510e576496ab56180c13f1648e7_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:59,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/9587c69220f9446286d9b675b296e0b9, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:38:59,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/9587c69220f9446286d9b675b296e0b9 is 175, key is test_row_0/A:col10/1731494338642/Put/seqid=0 2024-11-13T10:38:59,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742037_1213 (size=31255) 2024-11-13T10:38:59,733 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/9587c69220f9446286d9b675b296e0b9 2024-11-13T10:38:59,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/6d09f5a7cd7443d18489ac8a69ac8a67 is 50, key is test_row_0/B:col10/1731494338642/Put/seqid=0 2024-11-13T10:38:59,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742038_1214 (size=12301) 2024-11-13T10:38:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:38:59,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:38:59,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:38:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494399904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:38:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:38:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:00,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494400020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:00,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:00,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:00,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T10:39:00,159 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/6d09f5a7cd7443d18489ac8a69ac8a67 2024-11-13T10:39:00,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/2b95a31538e9453b967231ab02efb8e6 is 50, key is test_row_0/C:col10/1731494338642/Put/seqid=0 2024-11-13T10:39:00,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742039_1215 (size=12301) 2024-11-13T10:39:00,173 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/2b95a31538e9453b967231ab02efb8e6 2024-11-13T10:39:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/9587c69220f9446286d9b675b296e0b9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/9587c69220f9446286d9b675b296e0b9 2024-11-13T10:39:00,186 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/9587c69220f9446286d9b675b296e0b9, entries=150, sequenceid=288, filesize=30.5 K 2024-11-13T10:39:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/6d09f5a7cd7443d18489ac8a69ac8a67 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/6d09f5a7cd7443d18489ac8a69ac8a67 2024-11-13T10:39:00,192 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/6d09f5a7cd7443d18489ac8a69ac8a67, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T10:39:00,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/2b95a31538e9453b967231ab02efb8e6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b95a31538e9453b967231ab02efb8e6 2024-11-13T10:39:00,197 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b95a31538e9453b967231ab02efb8e6, entries=150, sequenceid=288, filesize=12.0 K 2024-11-13T10:39:00,198 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 99adb18da1f9e813f4728ff7b8100fa3 in 603ms, sequenceid=288, compaction requested=true 2024-11-13T10:39:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-13T10:39:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-13T10:39:00,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-13T10:39:00,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3100 sec 2024-11-13T10:39:00,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3140 sec 2024-11-13T10:39:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:00,230 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:00,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T10:39:00,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131c5e221c84d84c65be278532688e799e_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494340228/Put/seqid=0 2024-11-13T10:39:00,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742040_1216 (size=14994) 2024-11-13T10:39:00,244 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:00,252 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131c5e221c84d84c65be278532688e799e_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c5e221c84d84c65be278532688e799e_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:00,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/87e123e1e193476f96bbbe2a8a28edb1, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:00,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/87e123e1e193476f96bbbe2a8a28edb1 is 175, key is test_row_0/A:col10/1731494340228/Put/seqid=0 2024-11-13T10:39:00,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742041_1217 (size=39949) 2024-11-13T10:39:00,264 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/87e123e1e193476f96bbbe2a8a28edb1 2024-11-13T10:39:00,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494400262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:00,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/9a0ca69d2c0a46ecbe4bfbd395af0d1a is 50, key is test_row_0/B:col10/1731494340228/Put/seqid=0 2024-11-13T10:39:00,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742042_1218 (size=12301) 2024-11-13T10:39:00,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494400376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:00,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494400587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:00,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/9a0ca69d2c0a46ecbe4bfbd395af0d1a 2024-11-13T10:39:00,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/3a14f06bfbdc4833b02408f247799e4a is 50, key is test_row_0/C:col10/1731494340228/Put/seqid=0 2024-11-13T10:39:00,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742043_1219 (size=12301) 2024-11-13T10:39:00,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/3a14f06bfbdc4833b02408f247799e4a 2024-11-13T10:39:00,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/87e123e1e193476f96bbbe2a8a28edb1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/87e123e1e193476f96bbbe2a8a28edb1 2024-11-13T10:39:00,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/87e123e1e193476f96bbbe2a8a28edb1, entries=200, sequenceid=315, filesize=39.0 K 2024-11-13T10:39:00,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/9a0ca69d2c0a46ecbe4bfbd395af0d1a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/9a0ca69d2c0a46ecbe4bfbd395af0d1a 2024-11-13T10:39:00,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/9a0ca69d2c0a46ecbe4bfbd395af0d1a, entries=150, sequenceid=315, filesize=12.0 K 2024-11-13T10:39:00,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/3a14f06bfbdc4833b02408f247799e4a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3a14f06bfbdc4833b02408f247799e4a 2024-11-13T10:39:00,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3a14f06bfbdc4833b02408f247799e4a, entries=150, sequenceid=315, filesize=12.0 K 2024-11-13T10:39:00,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 99adb18da1f9e813f4728ff7b8100fa3 in 522ms, sequenceid=315, compaction requested=true 2024-11-13T10:39:00,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:00,753 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:00,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:00,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:00,754 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:00,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:00,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:00,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:00,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:00,763 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142838 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:00,763 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/A is initiating minor compaction (all files) 2024-11-13T10:39:00,763 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/A in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:00,763 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/c5c79fe358cf4b13a62c759d9da142a8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ff27708ec77f41cea3aadbddb814f2c6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/9587c69220f9446286d9b675b296e0b9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/87e123e1e193476f96bbbe2a8a28edb1] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=139.5 K 2024-11-13T10:39:00,763 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:00,763 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/c5c79fe358cf4b13a62c759d9da142a8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ff27708ec77f41cea3aadbddb814f2c6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/9587c69220f9446286d9b675b296e0b9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/87e123e1e193476f96bbbe2a8a28edb1] 2024-11-13T10:39:00,763 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:00,764 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/B is initiating minor compaction (all files) 2024-11-13T10:39:00,764 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/B in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:00,764 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/cc63be313dce4a17ba9e236f8f5b15ed, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/e8f61e84dd034374bbdbfb84ea77e27e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/6d09f5a7cd7443d18489ac8a69ac8a67, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/9a0ca69d2c0a46ecbe4bfbd395af0d1a] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=48.5 K 2024-11-13T10:39:00,764 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting cc63be313dce4a17ba9e236f8f5b15ed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731494337194 2024-11-13T10:39:00,764 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c79fe358cf4b13a62c759d9da142a8, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731494337194 2024-11-13T10:39:00,765 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f61e84dd034374bbdbfb84ea77e27e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731494337245 2024-11-13T10:39:00,765 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff27708ec77f41cea3aadbddb814f2c6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731494337239 2024-11-13T10:39:00,767 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9587c69220f9446286d9b675b296e0b9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731494338638 2024-11-13T10:39:00,768 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87e123e1e193476f96bbbe2a8a28edb1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731494339852 2024-11-13T10:39:00,772 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d09f5a7cd7443d18489ac8a69ac8a67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731494338638 2024-11-13T10:39:00,773 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a0ca69d2c0a46ecbe4bfbd395af0d1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731494339891 2024-11-13T10:39:00,789 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:00,791 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#B#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:00,792 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/8a56a22029e347c9825e3b13774b3d7d is 50, key is test_row_0/B:col10/1731494340228/Put/seqid=0 2024-11-13T10:39:00,805 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138a76de30308146fa8f25b81d7edf5dff_99adb18da1f9e813f4728ff7b8100fa3 store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:00,808 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138a76de30308146fa8f25b81d7edf5dff_99adb18da1f9e813f4728ff7b8100fa3, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:00,808 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138a76de30308146fa8f25b81d7edf5dff_99adb18da1f9e813f4728ff7b8100fa3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:00,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742044_1220 (size=13017) 2024-11-13T10:39:00,858 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/8a56a22029e347c9825e3b13774b3d7d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8a56a22029e347c9825e3b13774b3d7d 2024-11-13T10:39:00,866 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/B of 99adb18da1f9e813f4728ff7b8100fa3 into 8a56a22029e347c9825e3b13774b3d7d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:00,866 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:00,867 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/B, priority=12, startTime=1731494340754; duration=0sec 2024-11-13T10:39:00,867 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:00,867 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:B 2024-11-13T10:39:00,867 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:00,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742045_1221 (size=4469) 2024-11-13T10:39:00,870 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#A#compaction#183 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:00,871 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/b04628f695c84d2098b79b6de0695b45 is 175, key is test_row_0/A:col10/1731494340228/Put/seqid=0 2024-11-13T10:39:00,874 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:00,874 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/C is initiating minor compaction (all files) 2024-11-13T10:39:00,874 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/C in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:00,874 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/eb826b923fa540e7900e680e02ea4ee5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/459338a211b043a2b1270ad4f15ab6b9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b95a31538e9453b967231ab02efb8e6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3a14f06bfbdc4833b02408f247799e4a] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=48.5 K 2024-11-13T10:39:00,875 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting eb826b923fa540e7900e680e02ea4ee5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731494337194 2024-11-13T10:39:00,876 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 459338a211b043a2b1270ad4f15ab6b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731494337245 2024-11-13T10:39:00,876 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b95a31538e9453b967231ab02efb8e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731494338638 2024-11-13T10:39:00,876 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a14f06bfbdc4833b02408f247799e4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731494339891 2024-11-13T10:39:00,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742046_1222 (size=31971) 2024-11-13T10:39:00,904 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/b04628f695c84d2098b79b6de0695b45 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b04628f695c84d2098b79b6de0695b45 2024-11-13T10:39:00,911 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#C#compaction#185 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:00,912 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/9b7f0428204846d695da80316f227eab is 50, key is test_row_0/C:col10/1731494340228/Put/seqid=0 2024-11-13T10:39:00,916 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/A of 99adb18da1f9e813f4728ff7b8100fa3 into b04628f695c84d2098b79b6de0695b45(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:00,916 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:00,916 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/A, priority=12, startTime=1731494340753; duration=0sec 2024-11-13T10:39:00,916 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:00,916 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:A 2024-11-13T10:39:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:00,918 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:00,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:39:00,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742047_1223 (size=13017) 2024-11-13T10:39:00,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111304687314ef3d4c41952834f995d0ac6f_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494340913/Put/seqid=0 2024-11-13T10:39:00,949 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/9b7f0428204846d695da80316f227eab as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/9b7f0428204846d695da80316f227eab 2024-11-13T10:39:00,956 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/C of 99adb18da1f9e813f4728ff7b8100fa3 into 9b7f0428204846d695da80316f227eab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:00,956 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:00,957 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/C, priority=12, startTime=1731494340754; duration=0sec 2024-11-13T10:39:00,957 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:00,957 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:C 2024-11-13T10:39:00,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742048_1224 (size=14994) 2024-11-13T10:39:00,974 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:00,978 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111304687314ef3d4c41952834f995d0ac6f_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304687314ef3d4c41952834f995d0ac6f_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:00,984 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/b33c9fdc2f8146b38ed551d9de48eda0, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:00,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/b33c9fdc2f8146b38ed551d9de48eda0 is 175, key is test_row_0/A:col10/1731494340913/Put/seqid=0 2024-11-13T10:39:01,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742049_1225 (size=39949) 2024-11-13T10:39:01,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494401072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:01,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:01,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494401189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:01,404 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/b33c9fdc2f8146b38ed551d9de48eda0 2024-11-13T10:39:01,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494401409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:01,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:01,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1a314f8f8a334815847b1f2c5453362f is 50, key is test_row_0/B:col10/1731494340913/Put/seqid=0 2024-11-13T10:39:01,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494401427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:01,436 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:01,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494401441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:01,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494401441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:01,442 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:01,442 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:01,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494401453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:01,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742050_1226 (size=12301) 2024-11-13T10:39:01,455 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:01,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494401717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:01,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:01,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1a314f8f8a334815847b1f2c5453362f 2024-11-13T10:39:01,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/d5421877559541febd623386f16cb722 is 50, key is test_row_0/C:col10/1731494340913/Put/seqid=0 2024-11-13T10:39:01,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742051_1227 (size=12301) 2024-11-13T10:39:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T10:39:02,048 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:02,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-13T10:39:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T10:39:02,051 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:02,052 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:02,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T10:39:02,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T10:39:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:39:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:02,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:02,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:02,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494402227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:02,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/d5421877559541febd623386f16cb722 2024-11-13T10:39:02,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/b33c9fdc2f8146b38ed551d9de48eda0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b33c9fdc2f8146b38ed551d9de48eda0 2024-11-13T10:39:02,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b33c9fdc2f8146b38ed551d9de48eda0, entries=200, sequenceid=328, filesize=39.0 K 2024-11-13T10:39:02,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1a314f8f8a334815847b1f2c5453362f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1a314f8f8a334815847b1f2c5453362f 2024-11-13T10:39:02,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1a314f8f8a334815847b1f2c5453362f, entries=150, sequenceid=328, filesize=12.0 K 2024-11-13T10:39:02,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/d5421877559541febd623386f16cb722 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d5421877559541febd623386f16cb722 2024-11-13T10:39:02,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d5421877559541febd623386f16cb722, entries=150, sequenceid=328, filesize=12.0 K 2024-11-13T10:39:02,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 99adb18da1f9e813f4728ff7b8100fa3 in 1398ms, sequenceid=328, compaction requested=false 2024-11-13T10:39:02,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:02,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T10:39:02,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:02,360 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:39:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T10:39:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371a3b264e3df4b8f8bb1638e07a249f0_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494341023/Put/seqid=0 2024-11-13T10:39:02,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742052_1228 (size=12454) 2024-11-13T10:39:02,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,390 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371a3b264e3df4b8f8bb1638e07a249f0_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371a3b264e3df4b8f8bb1638e07a249f0_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:02,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/2f29f918bbe54cde89b384bccbe2e35d, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:02,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/2f29f918bbe54cde89b384bccbe2e35d is 175, key is test_row_0/A:col10/1731494341023/Put/seqid=0 2024-11-13T10:39:02,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742053_1229 (size=31255) 2024-11-13T10:39:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T10:39:02,797 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/2f29f918bbe54cde89b384bccbe2e35d 2024-11-13T10:39:02,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/8bee61dd6bbe41d4b40f66adbd9db64b is 50, key is test_row_0/B:col10/1731494341023/Put/seqid=0 2024-11-13T10:39:02,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742054_1230 (size=12301) 2024-11-13T10:39:02,841 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/8bee61dd6bbe41d4b40f66adbd9db64b 2024-11-13T10:39:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/51470b3b380c403fbe37f52fba7f14df is 50, key is test_row_0/C:col10/1731494341023/Put/seqid=0 2024-11-13T10:39:02,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742055_1231 (size=12301) 2024-11-13T10:39:02,893 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/51470b3b380c403fbe37f52fba7f14df 2024-11-13T10:39:02,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/2f29f918bbe54cde89b384bccbe2e35d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2f29f918bbe54cde89b384bccbe2e35d 2024-11-13T10:39:02,909 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2f29f918bbe54cde89b384bccbe2e35d, entries=150, sequenceid=354, filesize=30.5 K 2024-11-13T10:39:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/8bee61dd6bbe41d4b40f66adbd9db64b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8bee61dd6bbe41d4b40f66adbd9db64b 2024-11-13T10:39:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,918 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8bee61dd6bbe41d4b40f66adbd9db64b, entries=150, sequenceid=354, filesize=12.0 K 2024-11-13T10:39:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/51470b3b380c403fbe37f52fba7f14df as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/51470b3b380c403fbe37f52fba7f14df 2024-11-13T10:39:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,929 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/51470b3b380c403fbe37f52fba7f14df, entries=150, sequenceid=354, filesize=12.0 K 2024-11-13T10:39:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,930 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 99adb18da1f9e813f4728ff7b8100fa3 in 570ms, sequenceid=354, compaction requested=true 2024-11-13T10:39:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:02,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-13T10:39:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-13T10:39:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-13T10:39:02,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 889 msec 2024-11-13T10:39:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 898 msec 2024-11-13T10:39:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T10:39:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,187 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-13T10:39:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T10:39:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,190 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:03,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,191 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:03,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T10:39:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T10:39:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:03,346 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-13T10:39:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d51ebc95598e4ce38f14186cf66b5f4d_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_2/A:col10/1731494343240/Put/seqid=0 2024-11-13T10:39:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742056_1232 (size=7374) 2024-11-13T10:39:03,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:03,453 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d51ebc95598e4ce38f14186cf66b5f4d_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d51ebc95598e4ce38f14186cf66b5f4d_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:03,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. as already flushing 2024-11-13T10:39:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:03,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/1554b273f4ac4d5ab681448a0a387996, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:03,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/1554b273f4ac4d5ab681448a0a387996 is 175, key is test_row_2/A:col10/1731494343240/Put/seqid=0 2024-11-13T10:39:03,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742057_1233 (size=13865) 2024-11-13T10:39:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T10:39:03,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494403633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:03,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:03,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494403747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T10:39:03,877 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/1554b273f4ac4d5ab681448a0a387996 2024-11-13T10:39:03,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1140cfd3cc314f6ba809ce47aa9e8219 is 50, key is test_row_2/B:col10/1731494343240/Put/seqid=0 2024-11-13T10:39:03,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742058_1234 (size=7415) 2024-11-13T10:39:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494403956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:04,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494404266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:04,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:04,299 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1140cfd3cc314f6ba809ce47aa9e8219 2024-11-13T10:39:04,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/5cc2d9094e73464d83ee2ee7171bb8c8 is 50, key is test_row_2/C:col10/1731494343240/Put/seqid=0 2024-11-13T10:39:04,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742059_1235 (size=7415) 2024-11-13T10:39:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T10:39:04,729 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/5cc2d9094e73464d83ee2ee7171bb8c8 2024-11-13T10:39:04,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/1554b273f4ac4d5ab681448a0a387996 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/1554b273f4ac4d5ab681448a0a387996 2024-11-13T10:39:04,741 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/1554b273f4ac4d5ab681448a0a387996, entries=50, sequenceid=358, filesize=13.5 K 2024-11-13T10:39:04,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/1140cfd3cc314f6ba809ce47aa9e8219 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1140cfd3cc314f6ba809ce47aa9e8219 2024-11-13T10:39:04,747 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1140cfd3cc314f6ba809ce47aa9e8219, entries=50, sequenceid=358, filesize=7.2 K 2024-11-13T10:39:04,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/5cc2d9094e73464d83ee2ee7171bb8c8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/5cc2d9094e73464d83ee2ee7171bb8c8 2024-11-13T10:39:04,759 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/5cc2d9094e73464d83ee2ee7171bb8c8, entries=50, sequenceid=358, filesize=7.2 K 2024-11-13T10:39:04,761 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=194.56 KB/199230 for 99adb18da1f9e813f4728ff7b8100fa3 in 1415ms, sequenceid=358, compaction requested=true 2024-11-13T10:39:04,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:04,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:04,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-13T10:39:04,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-13T10:39:04,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-13T10:39:04,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5720 sec 2024-11-13T10:39:04,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5820 sec 2024-11-13T10:39:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:04,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-13T10:39:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494404796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:04,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113263296a5ef9244c0889ec9be55929a58_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494343604/Put/seqid=0 2024-11-13T10:39:04,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742060_1236 (size=12454) 2024-11-13T10:39:04,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494404913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:05,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494405126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:05,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:05,301 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:05,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113263296a5ef9244c0889ec9be55929a58_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113263296a5ef9244c0889ec9be55929a58_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:05,312 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/6de49b82892341279458e2ba1eb84138, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:05,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/6de49b82892341279458e2ba1eb84138 is 175, key is test_row_0/A:col10/1731494343604/Put/seqid=0 2024-11-13T10:39:05,334 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-11-13T10:39:05,334 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:05,334 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:05,335 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:05,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T10:39:05,337 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:05,338 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-13T10:39:05,338 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:05,338 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:05,338 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:05,338 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-13T10:39:05,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:05,338 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:05,338 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:05,338 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:05,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:05,340 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-13T10:39:05,340 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:05,340 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:05,340 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:05,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:05,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742061_1237 (size=31255) 2024-11-13T10:39:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494405437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:05,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37344 deadline: 1731494405457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:05,459 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:05,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37338 deadline: 1731494405467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:05,467 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:05,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37302 deadline: 1731494405477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:05,478 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:05,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37276 deadline: 1731494405487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:05,488 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in 99adb18da1f9e813f4728ff7b8100fa3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:05,744 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=67.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/6de49b82892341279458e2ba1eb84138 2024-11-13T10:39:05,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/40b99fcfc8a64cef8c91bb8e77c9377a is 50, key is test_row_0/B:col10/1731494343604/Put/seqid=0 2024-11-13T10:39:05,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742062_1238 (size=12301) 2024-11-13T10:39:05,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37290 deadline: 1731494405947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:06,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/40b99fcfc8a64cef8c91bb8e77c9377a 2024-11-13T10:39:06,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/3f32f6bc1ef84850a1b5ef1c45ebfd48 is 50, key is test_row_0/C:col10/1731494343604/Put/seqid=0 2024-11-13T10:39:06,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742063_1239 (size=12301) 2024-11-13T10:39:06,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/3f32f6bc1ef84850a1b5ef1c45ebfd48 2024-11-13T10:39:06,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/6de49b82892341279458e2ba1eb84138 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6de49b82892341279458e2ba1eb84138 2024-11-13T10:39:06,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6de49b82892341279458e2ba1eb84138, entries=150, sequenceid=391, filesize=30.5 K 2024-11-13T10:39:06,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/40b99fcfc8a64cef8c91bb8e77c9377a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/40b99fcfc8a64cef8c91bb8e77c9377a 2024-11-13T10:39:06,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/40b99fcfc8a64cef8c91bb8e77c9377a, entries=150, sequenceid=391, filesize=12.0 K 2024-11-13T10:39:06,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/3f32f6bc1ef84850a1b5ef1c45ebfd48 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3f32f6bc1ef84850a1b5ef1c45ebfd48 2024-11-13T10:39:06,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3f32f6bc1ef84850a1b5ef1c45ebfd48, entries=150, sequenceid=391, filesize=12.0 K 2024-11-13T10:39:06,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 99adb18da1f9e813f4728ff7b8100fa3 in 1794ms, sequenceid=391, compaction requested=true 2024-11-13T10:39:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:06,590 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T10:39:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 99adb18da1f9e813f4728ff7b8100fa3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:06,590 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T10:39:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:06,592 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57335 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T10:39:06,592 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 148295 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T10:39:06,592 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/A is initiating minor compaction (all files) 2024-11-13T10:39:06,592 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/B is initiating minor compaction (all files) 2024-11-13T10:39:06,592 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/A in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:06,592 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/B in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:06,592 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b04628f695c84d2098b79b6de0695b45, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b33c9fdc2f8146b38ed551d9de48eda0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2f29f918bbe54cde89b384bccbe2e35d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/1554b273f4ac4d5ab681448a0a387996, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6de49b82892341279458e2ba1eb84138] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=144.8 K 2024-11-13T10:39:06,592 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8a56a22029e347c9825e3b13774b3d7d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1a314f8f8a334815847b1f2c5453362f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8bee61dd6bbe41d4b40f66adbd9db64b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1140cfd3cc314f6ba809ce47aa9e8219, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/40b99fcfc8a64cef8c91bb8e77c9377a] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=56.0 K 2024-11-13T10:39:06,592 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:06,592 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b04628f695c84d2098b79b6de0695b45, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b33c9fdc2f8146b38ed551d9de48eda0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2f29f918bbe54cde89b384bccbe2e35d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/1554b273f4ac4d5ab681448a0a387996, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6de49b82892341279458e2ba1eb84138] 2024-11-13T10:39:06,592 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a56a22029e347c9825e3b13774b3d7d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731494339891 2024-11-13T10:39:06,593 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04628f695c84d2098b79b6de0695b45, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731494339891 2024-11-13T10:39:06,593 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a314f8f8a334815847b1f2c5453362f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731494340238 2024-11-13T10:39:06,593 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b33c9fdc2f8146b38ed551d9de48eda0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731494340238 2024-11-13T10:39:06,593 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bee61dd6bbe41d4b40f66adbd9db64b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731494341023 2024-11-13T10:39:06,593 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f29f918bbe54cde89b384bccbe2e35d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731494341023 2024-11-13T10:39:06,594 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1140cfd3cc314f6ba809ce47aa9e8219, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731494343240 2024-11-13T10:39:06,594 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1554b273f4ac4d5ab681448a0a387996, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731494343240 2024-11-13T10:39:06,594 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b99fcfc8a64cef8c91bb8e77c9377a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731494343604 2024-11-13T10:39:06,594 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6de49b82892341279458e2ba1eb84138, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731494343604 2024-11-13T10:39:06,610 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#B#compaction#198 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:06,611 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/9c024c725c8d40cdb17b6564f02f8fa3 is 50, key is test_row_0/B:col10/1731494343604/Put/seqid=0 2024-11-13T10:39:06,613 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:06,622 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138aa5709b36c94416b19e6fd8d0b07edf_99adb18da1f9e813f4728ff7b8100fa3 store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:06,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742064_1240 (size=13187) 2024-11-13T10:39:06,644 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138aa5709b36c94416b19e6fd8d0b07edf_99adb18da1f9e813f4728ff7b8100fa3, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:06,644 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138aa5709b36c94416b19e6fd8d0b07edf_99adb18da1f9e813f4728ff7b8100fa3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:06,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742065_1241 (size=4469) 2024-11-13T10:39:06,653 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#A#compaction#199 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:06,654 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/705d1d03450e4b95b02d42227edc06fd is 175, key is test_row_0/A:col10/1731494343604/Put/seqid=0 2024-11-13T10:39:06,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742066_1242 (size=32141) 2024-11-13T10:39:06,958 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-13T10:39:06,958 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:06,958 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:06,958 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:06,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:07,031 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/9c024c725c8d40cdb17b6564f02f8fa3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/9c024c725c8d40cdb17b6564f02f8fa3 2024-11-13T10:39:07,035 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/B of 99adb18da1f9e813f4728ff7b8100fa3 into 9c024c725c8d40cdb17b6564f02f8fa3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:07,035 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:07,035 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/B, priority=11, startTime=1731494346590; duration=0sec 2024-11-13T10:39:07,035 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:07,035 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:B 2024-11-13T10:39:07,035 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T10:39:07,036 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57335 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T10:39:07,037 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 99adb18da1f9e813f4728ff7b8100fa3/C is initiating minor compaction (all files) 2024-11-13T10:39:07,037 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 99adb18da1f9e813f4728ff7b8100fa3/C in TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:07,037 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/9b7f0428204846d695da80316f227eab, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d5421877559541febd623386f16cb722, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/51470b3b380c403fbe37f52fba7f14df, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/5cc2d9094e73464d83ee2ee7171bb8c8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3f32f6bc1ef84850a1b5ef1c45ebfd48] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp, totalSize=56.0 K 2024-11-13T10:39:07,037 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b7f0428204846d695da80316f227eab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731494339891 2024-11-13T10:39:07,037 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting d5421877559541febd623386f16cb722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731494340238 2024-11-13T10:39:07,037 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 51470b3b380c403fbe37f52fba7f14df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731494341023 2024-11-13T10:39:07,038 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cc2d9094e73464d83ee2ee7171bb8c8, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731494343240 2024-11-13T10:39:07,038 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f32f6bc1ef84850a1b5ef1c45ebfd48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731494343604 2024-11-13T10:39:07,046 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 99adb18da1f9e813f4728ff7b8100fa3#C#compaction#200 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:07,046 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/9fd86d43a49949c6afe6d83d6aad94c0 is 50, key is test_row_0/C:col10/1731494343604/Put/seqid=0 2024-11-13T10:39:07,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742067_1243 (size=13187) 2024-11-13T10:39:07,062 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/705d1d03450e4b95b02d42227edc06fd as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/705d1d03450e4b95b02d42227edc06fd 2024-11-13T10:39:07,066 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/A of 99adb18da1f9e813f4728ff7b8100fa3 into 705d1d03450e4b95b02d42227edc06fd(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:07,066 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:07,066 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/A, priority=11, startTime=1731494346590; duration=0sec 2024-11-13T10:39:07,066 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:07,066 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:A 2024-11-13T10:39:07,455 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/9fd86d43a49949c6afe6d83d6aad94c0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/9fd86d43a49949c6afe6d83d6aad94c0 2024-11-13T10:39:07,460 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 99adb18da1f9e813f4728ff7b8100fa3/C of 99adb18da1f9e813f4728ff7b8100fa3 into 9fd86d43a49949c6afe6d83d6aad94c0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:07,460 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 99adb18da1f9e813f4728ff7b8100fa3: 2024-11-13T10:39:07,460 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., storeName=99adb18da1f9e813f4728ff7b8100fa3/C, priority=11, startTime=1731494346590; duration=0sec 2024-11-13T10:39:07,460 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:07,460 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 99adb18da1f9e813f4728ff7b8100fa3:C 2024-11-13T10:39:11,029 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T10:39:15,488 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-13T10:39:15,488 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:15,488 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:15,488 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:15,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:15,508 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-13T10:39:15,508 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:15,508 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:15,508 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:15,508 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:15,529 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-13T10:39:15,529 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:15,529 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:15,529 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:15,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:15,568 INFO [Thread-669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-669. 2024-11-13T10:39:15,568 DEBUG [Thread-669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:15,568 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:15,568 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:15,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2569 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2526 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1159 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3476 rows 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1175 2024-11-13T10:39:15,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3525 rows 2024-11-13T10:39:15,569 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T10:39:15,569 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T10:39:15,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:15,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:15,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:15,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T10:39:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T10:39:15,572 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494355572"}]},"ts":"1731494355572"} 2024-11-13T10:39:15,574 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T10:39:15,574 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T10:39:15,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T10:39:15,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, UNASSIGN}] 2024-11-13T10:39:15,576 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, UNASSIGN 2024-11-13T10:39:15,577 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=99adb18da1f9e813f4728ff7b8100fa3, regionState=CLOSING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:39:15,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, UNASSIGN because future has completed 2024-11-13T10:39:15,578 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T10:39:15,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:39:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T10:39:15,730 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:15,730 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T10:39:15,730 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 99adb18da1f9e813f4728ff7b8100fa3, disabling compactions & flushes 2024-11-13T10:39:15,730 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:15,730 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:15,730 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. after waiting 0 ms 2024-11-13T10:39:15,730 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:15,730 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 99adb18da1f9e813f4728ff7b8100fa3 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T10:39:15,737 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f379304fca194f5bb1c8d74b6e3babba_99adb18da1f9e813f4728ff7b8100fa3 is 50, key is test_row_0/A:col10/1731494355507/Put/seqid=0 2024-11-13T10:39:15,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742068_1244 (size=9914) 2024-11-13T10:39:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T10:39:16,144 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:16,163 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f379304fca194f5bb1c8d74b6e3babba_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f379304fca194f5bb1c8d74b6e3babba_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,165 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/164ab076d94849bd82adea34313ba964, store: [table=TestAcidGuarantees family=A region=99adb18da1f9e813f4728ff7b8100fa3] 2024-11-13T10:39:16,166 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/164ab076d94849bd82adea34313ba964 is 175, key is test_row_0/A:col10/1731494355507/Put/seqid=0 2024-11-13T10:39:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742069_1245 (size=22561) 2024-11-13T10:39:16,176 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/164ab076d94849bd82adea34313ba964 2024-11-13T10:39:16,186 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/3677898f6e234d498c18d5aee6514588 is 50, key is test_row_0/B:col10/1731494355507/Put/seqid=0 2024-11-13T10:39:16,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742070_1246 (size=9857) 2024-11-13T10:39:16,192 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/3677898f6e234d498c18d5aee6514588 2024-11-13T10:39:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T10:39:16,219 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/48568748708740128541d9f4807d73c3 is 50, key is test_row_0/C:col10/1731494355507/Put/seqid=0 2024-11-13T10:39:16,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742071_1247 (size=9857) 2024-11-13T10:39:16,235 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/48568748708740128541d9f4807d73c3 2024-11-13T10:39:16,240 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/A/164ab076d94849bd82adea34313ba964 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/164ab076d94849bd82adea34313ba964 2024-11-13T10:39:16,246 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/164ab076d94849bd82adea34313ba964, entries=100, sequenceid=402, filesize=22.0 K 2024-11-13T10:39:16,248 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/B/3677898f6e234d498c18d5aee6514588 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/3677898f6e234d498c18d5aee6514588 2024-11-13T10:39:16,252 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/3677898f6e234d498c18d5aee6514588, entries=100, sequenceid=402, filesize=9.6 K 2024-11-13T10:39:16,253 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/.tmp/C/48568748708740128541d9f4807d73c3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/48568748708740128541d9f4807d73c3 2024-11-13T10:39:16,257 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/48568748708740128541d9f4807d73c3, entries=100, sequenceid=402, filesize=9.6 K 2024-11-13T10:39:16,258 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 99adb18da1f9e813f4728ff7b8100fa3 in 528ms, sequenceid=402, compaction requested=false 2024-11-13T10:39:16,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a7faabe32a884e33a5cbb93384aa87d9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/da7a5e090f644d97a1b160e454485642, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/13f4b96d6bd74fb0a68a85ad68b93ad3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2b4a0ce805e6432eb2520c09ca8e5cd4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ab7e667d0e724d1fa631a86f800f6097, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/35aa7828c2ee4c948a9440edec25fcaf, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/7cf248e0ccf84146a455890b71f8570a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/792c304b638840698da5f5b5d90599ce, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6c3cb5125d264ee3ad9901a9bad1b2e0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/e6f96b9636e044eca6beb67741846b6e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/bbf3f01193634d918e15ad17c38077a1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a389d6bcab3b4d33b744a4e7cffcbd56, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/d567d4080e904760b4ca4611b7fbf486, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ffd6abc6abf34481bc24453af2e14f02, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/3008f4de6bb24218943adc347ccff761, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/95ee2aa041774f8daabc47b201e38805, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/caae05da88104d10adf4b94895f1bf4b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/c5c79fe358cf4b13a62c759d9da142a8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ff27708ec77f41cea3aadbddb814f2c6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/9587c69220f9446286d9b675b296e0b9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/87e123e1e193476f96bbbe2a8a28edb1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b04628f695c84d2098b79b6de0695b45, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b33c9fdc2f8146b38ed551d9de48eda0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2f29f918bbe54cde89b384bccbe2e35d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/1554b273f4ac4d5ab681448a0a387996, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6de49b82892341279458e2ba1eb84138] to archive 2024-11-13T10:39:16,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:39:16,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a7faabe32a884e33a5cbb93384aa87d9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a7faabe32a884e33a5cbb93384aa87d9 2024-11-13T10:39:16,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/da7a5e090f644d97a1b160e454485642 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/da7a5e090f644d97a1b160e454485642 2024-11-13T10:39:16,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/13f4b96d6bd74fb0a68a85ad68b93ad3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/13f4b96d6bd74fb0a68a85ad68b93ad3 2024-11-13T10:39:16,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2b4a0ce805e6432eb2520c09ca8e5cd4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2b4a0ce805e6432eb2520c09ca8e5cd4 2024-11-13T10:39:16,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ab7e667d0e724d1fa631a86f800f6097 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ab7e667d0e724d1fa631a86f800f6097 2024-11-13T10:39:16,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/35aa7828c2ee4c948a9440edec25fcaf to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/35aa7828c2ee4c948a9440edec25fcaf 2024-11-13T10:39:16,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/7cf248e0ccf84146a455890b71f8570a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/7cf248e0ccf84146a455890b71f8570a 2024-11-13T10:39:16,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/792c304b638840698da5f5b5d90599ce to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/792c304b638840698da5f5b5d90599ce 2024-11-13T10:39:16,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6c3cb5125d264ee3ad9901a9bad1b2e0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6c3cb5125d264ee3ad9901a9bad1b2e0 2024-11-13T10:39:16,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/e6f96b9636e044eca6beb67741846b6e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/e6f96b9636e044eca6beb67741846b6e 2024-11-13T10:39:16,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/bbf3f01193634d918e15ad17c38077a1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/bbf3f01193634d918e15ad17c38077a1 2024-11-13T10:39:16,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a389d6bcab3b4d33b744a4e7cffcbd56 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/a389d6bcab3b4d33b744a4e7cffcbd56 2024-11-13T10:39:16,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/d567d4080e904760b4ca4611b7fbf486 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/d567d4080e904760b4ca4611b7fbf486 2024-11-13T10:39:16,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ffd6abc6abf34481bc24453af2e14f02 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ffd6abc6abf34481bc24453af2e14f02 2024-11-13T10:39:16,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/3008f4de6bb24218943adc347ccff761 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/3008f4de6bb24218943adc347ccff761 2024-11-13T10:39:16,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/95ee2aa041774f8daabc47b201e38805 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/95ee2aa041774f8daabc47b201e38805 2024-11-13T10:39:16,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/caae05da88104d10adf4b94895f1bf4b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/caae05da88104d10adf4b94895f1bf4b 2024-11-13T10:39:16,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/c5c79fe358cf4b13a62c759d9da142a8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/c5c79fe358cf4b13a62c759d9da142a8 2024-11-13T10:39:16,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ff27708ec77f41cea3aadbddb814f2c6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/ff27708ec77f41cea3aadbddb814f2c6 2024-11-13T10:39:16,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/9587c69220f9446286d9b675b296e0b9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/9587c69220f9446286d9b675b296e0b9 2024-11-13T10:39:16,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/87e123e1e193476f96bbbe2a8a28edb1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/87e123e1e193476f96bbbe2a8a28edb1 2024-11-13T10:39:16,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b04628f695c84d2098b79b6de0695b45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b04628f695c84d2098b79b6de0695b45 2024-11-13T10:39:16,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b33c9fdc2f8146b38ed551d9de48eda0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/b33c9fdc2f8146b38ed551d9de48eda0 2024-11-13T10:39:16,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2f29f918bbe54cde89b384bccbe2e35d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/2f29f918bbe54cde89b384bccbe2e35d 2024-11-13T10:39:16,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/1554b273f4ac4d5ab681448a0a387996 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/1554b273f4ac4d5ab681448a0a387996 2024-11-13T10:39:16,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6de49b82892341279458e2ba1eb84138 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/6de49b82892341279458e2ba1eb84138 2024-11-13T10:39:16,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/d66b379e86c94199bd654457cf53bf3b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/bd7c808b0b384c919e0b14cb9618fc7c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1cc2af0f47544cc8830f48cd0f0142e9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1e5742c3f9f94fb3b024a031df675603, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8d0fc3d180bd44dd814c4104cd39c371, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f5cebf2829874cd6ae73a360600276d9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/72fa98524732441c8cf2539b92251473, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f83966d94d9345cf97bae8b9e66af4c4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/7d0681a106e24ef2984ee78b695e961b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/87f1c05febd9491fa1c37d201439ee1a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b79325a6cfb4472492eff1e92efedc1f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/11b2dfc5a296473cb6c77c7c59a529bf, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b56c475b21ea4ade8bdda47eb392a72a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/499dab5194a44cf7af07461bba1d0785, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/24e87b781840497caf7d0aa70f295e12, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/64875e6e959445d096ea415f95d91871, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/cc63be313dce4a17ba9e236f8f5b15ed, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/34c4e87f67a14ec880e270d2cd064287, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/e8f61e84dd034374bbdbfb84ea77e27e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/6d09f5a7cd7443d18489ac8a69ac8a67, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8a56a22029e347c9825e3b13774b3d7d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/9a0ca69d2c0a46ecbe4bfbd395af0d1a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1a314f8f8a334815847b1f2c5453362f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8bee61dd6bbe41d4b40f66adbd9db64b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1140cfd3cc314f6ba809ce47aa9e8219, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/40b99fcfc8a64cef8c91bb8e77c9377a] to archive 2024-11-13T10:39:16,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:39:16,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/d66b379e86c94199bd654457cf53bf3b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/d66b379e86c94199bd654457cf53bf3b 2024-11-13T10:39:16,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/bd7c808b0b384c919e0b14cb9618fc7c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/bd7c808b0b384c919e0b14cb9618fc7c 2024-11-13T10:39:16,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1cc2af0f47544cc8830f48cd0f0142e9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1cc2af0f47544cc8830f48cd0f0142e9 2024-11-13T10:39:16,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1e5742c3f9f94fb3b024a031df675603 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1e5742c3f9f94fb3b024a031df675603 2024-11-13T10:39:16,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8d0fc3d180bd44dd814c4104cd39c371 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8d0fc3d180bd44dd814c4104cd39c371 2024-11-13T10:39:16,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f5cebf2829874cd6ae73a360600276d9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f5cebf2829874cd6ae73a360600276d9 2024-11-13T10:39:16,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/72fa98524732441c8cf2539b92251473 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/72fa98524732441c8cf2539b92251473 2024-11-13T10:39:16,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f83966d94d9345cf97bae8b9e66af4c4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/f83966d94d9345cf97bae8b9e66af4c4 2024-11-13T10:39:16,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/7d0681a106e24ef2984ee78b695e961b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/7d0681a106e24ef2984ee78b695e961b 2024-11-13T10:39:16,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/87f1c05febd9491fa1c37d201439ee1a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/87f1c05febd9491fa1c37d201439ee1a 2024-11-13T10:39:16,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b79325a6cfb4472492eff1e92efedc1f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b79325a6cfb4472492eff1e92efedc1f 2024-11-13T10:39:16,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/11b2dfc5a296473cb6c77c7c59a529bf to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/11b2dfc5a296473cb6c77c7c59a529bf 2024-11-13T10:39:16,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b56c475b21ea4ade8bdda47eb392a72a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/b56c475b21ea4ade8bdda47eb392a72a 2024-11-13T10:39:16,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/499dab5194a44cf7af07461bba1d0785 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/499dab5194a44cf7af07461bba1d0785 2024-11-13T10:39:16,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/24e87b781840497caf7d0aa70f295e12 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/24e87b781840497caf7d0aa70f295e12 2024-11-13T10:39:16,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/64875e6e959445d096ea415f95d91871 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/64875e6e959445d096ea415f95d91871 2024-11-13T10:39:16,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/cc63be313dce4a17ba9e236f8f5b15ed to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/cc63be313dce4a17ba9e236f8f5b15ed 2024-11-13T10:39:16,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/34c4e87f67a14ec880e270d2cd064287 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/34c4e87f67a14ec880e270d2cd064287 2024-11-13T10:39:16,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/e8f61e84dd034374bbdbfb84ea77e27e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/e8f61e84dd034374bbdbfb84ea77e27e 2024-11-13T10:39:16,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/6d09f5a7cd7443d18489ac8a69ac8a67 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/6d09f5a7cd7443d18489ac8a69ac8a67 2024-11-13T10:39:16,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8a56a22029e347c9825e3b13774b3d7d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8a56a22029e347c9825e3b13774b3d7d 2024-11-13T10:39:16,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/9a0ca69d2c0a46ecbe4bfbd395af0d1a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/9a0ca69d2c0a46ecbe4bfbd395af0d1a 2024-11-13T10:39:16,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1a314f8f8a334815847b1f2c5453362f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1a314f8f8a334815847b1f2c5453362f 2024-11-13T10:39:16,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8bee61dd6bbe41d4b40f66adbd9db64b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/8bee61dd6bbe41d4b40f66adbd9db64b 2024-11-13T10:39:16,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1140cfd3cc314f6ba809ce47aa9e8219 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/1140cfd3cc314f6ba809ce47aa9e8219 2024-11-13T10:39:16,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/40b99fcfc8a64cef8c91bb8e77c9377a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/40b99fcfc8a64cef8c91bb8e77c9377a 2024-11-13T10:39:16,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/0df76054bf1a4c88a10b34d3daea153d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/1e8d5551f7474394b915ac87723837ca, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e31606752df141beb86069b9e76f18a8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/ce601d3e21114683b74a7d068b405ac0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/59fb59b3606a422f81683387e5af346f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/58e1eb70bdaa420685da8f19e98ace4a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d9d47381bdbf477e8a21d84677585125, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b9532aa8a1841ec8e3de42e6d1c53c7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/78327cf9924b4a47b075c3c6d8150242, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e6679fe0cfef4513b1c61f413f6f27c2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/7eb78e4936a9481f9f6c0e57564bdc05, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/931dc67b185342f980f7bef35fede76e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/26dfd1d9db4d4cf089d65f3db85a4f9c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/80ddb6df09ac4ed081f9324007c75ae9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/03e4915e7506469586415cd74b831f68, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/bf8d54812ce24d55a28b7d2e39f62022, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/eb826b923fa540e7900e680e02ea4ee5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/4ea3363b796c486a848343e37581e5a8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/459338a211b043a2b1270ad4f15ab6b9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b95a31538e9453b967231ab02efb8e6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/9b7f0428204846d695da80316f227eab, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3a14f06bfbdc4833b02408f247799e4a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d5421877559541febd623386f16cb722, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/51470b3b380c403fbe37f52fba7f14df, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/5cc2d9094e73464d83ee2ee7171bb8c8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3f32f6bc1ef84850a1b5ef1c45ebfd48] to archive 2024-11-13T10:39:16,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:39:16,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/0df76054bf1a4c88a10b34d3daea153d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/0df76054bf1a4c88a10b34d3daea153d 2024-11-13T10:39:16,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/1e8d5551f7474394b915ac87723837ca to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/1e8d5551f7474394b915ac87723837ca 2024-11-13T10:39:16,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e31606752df141beb86069b9e76f18a8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e31606752df141beb86069b9e76f18a8 2024-11-13T10:39:16,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/ce601d3e21114683b74a7d068b405ac0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/ce601d3e21114683b74a7d068b405ac0 2024-11-13T10:39:16,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/59fb59b3606a422f81683387e5af346f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/59fb59b3606a422f81683387e5af346f 2024-11-13T10:39:16,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/58e1eb70bdaa420685da8f19e98ace4a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/58e1eb70bdaa420685da8f19e98ace4a 2024-11-13T10:39:16,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d9d47381bdbf477e8a21d84677585125 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d9d47381bdbf477e8a21d84677585125 2024-11-13T10:39:16,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b9532aa8a1841ec8e3de42e6d1c53c7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b9532aa8a1841ec8e3de42e6d1c53c7 2024-11-13T10:39:16,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/78327cf9924b4a47b075c3c6d8150242 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/78327cf9924b4a47b075c3c6d8150242 2024-11-13T10:39:16,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e6679fe0cfef4513b1c61f413f6f27c2 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/e6679fe0cfef4513b1c61f413f6f27c2 2024-11-13T10:39:16,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/7eb78e4936a9481f9f6c0e57564bdc05 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/7eb78e4936a9481f9f6c0e57564bdc05 2024-11-13T10:39:16,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/931dc67b185342f980f7bef35fede76e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/931dc67b185342f980f7bef35fede76e 2024-11-13T10:39:16,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/26dfd1d9db4d4cf089d65f3db85a4f9c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/26dfd1d9db4d4cf089d65f3db85a4f9c 2024-11-13T10:39:16,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/80ddb6df09ac4ed081f9324007c75ae9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/80ddb6df09ac4ed081f9324007c75ae9 2024-11-13T10:39:16,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/03e4915e7506469586415cd74b831f68 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/03e4915e7506469586415cd74b831f68 2024-11-13T10:39:16,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/bf8d54812ce24d55a28b7d2e39f62022 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/bf8d54812ce24d55a28b7d2e39f62022 2024-11-13T10:39:16,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/eb826b923fa540e7900e680e02ea4ee5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/eb826b923fa540e7900e680e02ea4ee5 2024-11-13T10:39:16,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/4ea3363b796c486a848343e37581e5a8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/4ea3363b796c486a848343e37581e5a8 2024-11-13T10:39:16,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/459338a211b043a2b1270ad4f15ab6b9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/459338a211b043a2b1270ad4f15ab6b9 2024-11-13T10:39:16,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b95a31538e9453b967231ab02efb8e6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/2b95a31538e9453b967231ab02efb8e6 2024-11-13T10:39:16,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/9b7f0428204846d695da80316f227eab to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/9b7f0428204846d695da80316f227eab 2024-11-13T10:39:16,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3a14f06bfbdc4833b02408f247799e4a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3a14f06bfbdc4833b02408f247799e4a 2024-11-13T10:39:16,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d5421877559541febd623386f16cb722 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/d5421877559541febd623386f16cb722 2024-11-13T10:39:16,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/51470b3b380c403fbe37f52fba7f14df to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/51470b3b380c403fbe37f52fba7f14df 2024-11-13T10:39:16,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/5cc2d9094e73464d83ee2ee7171bb8c8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/5cc2d9094e73464d83ee2ee7171bb8c8 2024-11-13T10:39:16,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3f32f6bc1ef84850a1b5ef1c45ebfd48 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/3f32f6bc1ef84850a1b5ef1c45ebfd48 2024-11-13T10:39:16,513 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/recovered.edits/405.seqid, newMaxSeqId=405, maxSeqId=4 2024-11-13T10:39:16,514 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. 2024-11-13T10:39:16,514 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 99adb18da1f9e813f4728ff7b8100fa3: Waiting for close lock at 1731494355730Running coprocessor pre-close hooks at 1731494355730Disabling compacts and flushes for region at 1731494355730Disabling writes for close at 1731494355730Obtaining lock to block concurrent updates at 1731494355730Preparing flush snapshotting stores in 99adb18da1f9e813f4728ff7b8100fa3 at 1731494355730Finished memstore snapshotting TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731494355731 (+1 ms)Flushing stores of TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3. at 1731494355731Flushing 99adb18da1f9e813f4728ff7b8100fa3/A: creating writer at 1731494355731Flushing mob file 99adb18da1f9e813f4728ff7b8100fa3/A: appending metadata at 1731494355736 (+5 ms)Flushing mob file 99adb18da1f9e813f4728ff7b8100fa3/A: closing flushed file at 1731494355736Flushing 99adb18da1f9e813f4728ff7b8100fa3/A: appending metadata at 1731494356165 (+429 ms)Flushing 99adb18da1f9e813f4728ff7b8100fa3/A: closing flushed file at 1731494356165Flushing 99adb18da1f9e813f4728ff7b8100fa3/B: creating writer at 1731494356181 (+16 ms)Flushing 99adb18da1f9e813f4728ff7b8100fa3/B: appending metadata at 1731494356185 (+4 ms)Flushing 99adb18da1f9e813f4728ff7b8100fa3/B: closing flushed file at 1731494356185Flushing 99adb18da1f9e813f4728ff7b8100fa3/C: creating writer at 1731494356214 (+29 ms)Flushing 99adb18da1f9e813f4728ff7b8100fa3/C: appending metadata at 1731494356218 (+4 ms)Flushing 99adb18da1f9e813f4728ff7b8100fa3/C: closing flushed file at 1731494356218Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@627222ab: reopening flushed file at 1731494356240 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f07323d: reopening flushed file at 1731494356246 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62390325: reopening flushed file at 1731494356252 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 99adb18da1f9e813f4728ff7b8100fa3 in 528ms, sequenceid=402, compaction requested=false at 1731494356258 (+6 ms)Writing region close event to WAL at 1731494356506 (+248 ms)Running coprocessor post-close hooks at 1731494356514 (+8 ms)Closed at 1731494356514 2024-11-13T10:39:16,516 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,517 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=99adb18da1f9e813f4728ff7b8100fa3, regionState=CLOSED 2024-11-13T10:39:16,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:39:16,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-13T10:39:16,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 99adb18da1f9e813f4728ff7b8100fa3, server=770665a7984d,39101,1731494292956 in 941 msec 2024-11-13T10:39:16,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-13T10:39:16,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=99adb18da1f9e813f4728ff7b8100fa3, UNASSIGN in 947 msec 2024-11-13T10:39:16,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-13T10:39:16,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 951 msec 2024-11-13T10:39:16,531 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494356530"}]},"ts":"1731494356530"} 2024-11-13T10:39:16,533 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T10:39:16,534 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T10:39:16,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 968 msec 2024-11-13T10:39:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T10:39:16,707 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:16,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T10:39:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:16,712 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:16,713 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T10:39:16,717 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,720 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/recovered.edits] 2024-11-13T10:39:16,726 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/164ab076d94849bd82adea34313ba964 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/164ab076d94849bd82adea34313ba964 2024-11-13T10:39:16,728 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/705d1d03450e4b95b02d42227edc06fd to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/A/705d1d03450e4b95b02d42227edc06fd 2024-11-13T10:39:16,732 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/3677898f6e234d498c18d5aee6514588 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/3677898f6e234d498c18d5aee6514588 2024-11-13T10:39:16,734 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/9c024c725c8d40cdb17b6564f02f8fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/B/9c024c725c8d40cdb17b6564f02f8fa3 2024-11-13T10:39:16,738 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/48568748708740128541d9f4807d73c3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/48568748708740128541d9f4807d73c3 2024-11-13T10:39:16,745 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/9fd86d43a49949c6afe6d83d6aad94c0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/C/9fd86d43a49949c6afe6d83d6aad94c0 2024-11-13T10:39:16,761 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/recovered.edits/405.seqid to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3/recovered.edits/405.seqid 2024-11-13T10:39:16,762 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,762 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T10:39:16,763 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T10:39:16,765 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T10:39:16,771 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304687314ef3d4c41952834f995d0ac6f_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111304687314ef3d4c41952834f995d0ac6f_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,773 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111305476245062e49f59ee3851707c20827_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111305476245062e49f59ee3851707c20827_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,779 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308035d01cd7f4dd68d9abd5d70278bcf_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308035d01cd7f4dd68d9abd5d70278bcf_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,781 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c5e221c84d84c65be278532688e799e_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131c5e221c84d84c65be278532688e799e_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,783 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113263296a5ef9244c0889ec9be55929a58_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113263296a5ef9244c0889ec9be55929a58_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,784 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111332628510e576496ab56180c13f1648e7_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111332628510e576496ab56180c13f1648e7_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,786 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111332ec3dc1bdc14675b8a22abecf6a3d30_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111332ec3dc1bdc14675b8a22abecf6a3d30_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,787 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111340201947582a4b9485c9f44e16ea69ed_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111340201947582a4b9485c9f44e16ea69ed_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,789 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371a3b264e3df4b8f8bb1638e07a249f0_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371a3b264e3df4b8f8bb1638e07a249f0_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,791 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111376934e01d58e4b119e07a4c308bd7422_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111376934e01d58e4b119e07a4c308bd7422_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,792 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aa42c83527914cf9ba7db01798ece3f6_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113aa42c83527914cf9ba7db01798ece3f6_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,793 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ae4c098dd4b74580950d0cb8cd29546a_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ae4c098dd4b74580950d0cb8cd29546a_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,795 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bcf3084cf76e4f79aff5eda6d4c36c56_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bcf3084cf76e4f79aff5eda6d4c36c56_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,796 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c2fcbab188da4064855c7dc280b3e038_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c2fcbab188da4064855c7dc280b3e038_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,797 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c5a43ef7bb594e3bb2758cef8e9046ba_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c5a43ef7bb594e3bb2758cef8e9046ba_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,798 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c69ad77a935642ee9c355ce389d91134_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c69ad77a935642ee9c355ce389d91134_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,800 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d51ebc95598e4ce38f14186cf66b5f4d_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113d51ebc95598e4ce38f14186cf66b5f4d_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,801 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e38981f6625848b9985b141060941eb9_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e38981f6625848b9985b141060941eb9_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,802 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ec6e602ee69a424bb1459b11905e8c7d_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ec6e602ee69a424bb1459b11905e8c7d_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,810 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f235afb74bd143f89872a8921f048f1f_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f235afb74bd143f89872a8921f048f1f_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,820 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f379304fca194f5bb1c8d74b6e3babba_99adb18da1f9e813f4728ff7b8100fa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f379304fca194f5bb1c8d74b6e3babba_99adb18da1f9e813f4728ff7b8100fa3 2024-11-13T10:39:16,821 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T10:39:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T10:39:16,829 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:16,835 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T10:39:16,838 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T10:39:16,840 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:16,840 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T10:39:16,840 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731494356840"}]},"ts":"9223372036854775807"} 2024-11-13T10:39:16,847 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T10:39:16,847 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 99adb18da1f9e813f4728ff7b8100fa3, NAME => 'TestAcidGuarantees,,1731494323089.99adb18da1f9e813f4728ff7b8100fa3.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T10:39:16,847 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T10:39:16,847 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731494356847"}]},"ts":"9223372036854775807"} 2024-11-13T10:39:16,850 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T10:39:16,850 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:16,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 142 msec 2024-11-13T10:39:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-13T10:39:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T10:39:17,037 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:17,050 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 238), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=851 (was 1000), ProcessCount=11 (was 11), AvailableMemoryMB=453 (was 641) 2024-11-13T10:39:17,064 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=851, ProcessCount=12, AvailableMemoryMB=453 2024-11-13T10:39:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T10:39:17,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T10:39:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:17,069 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T10:39:17,069 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:17,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-13T10:39:17,071 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T10:39:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-13T10:39:17,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742072_1248 (size=959) 2024-11-13T10:39:17,097 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617 2024-11-13T10:39:17,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742073_1249 (size=53) 2024-11-13T10:39:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-13T10:39:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-13T10:39:17,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:39:17,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 667b2b6bc191e21181054b3ac4dbe50b, disabling compactions & flushes 2024-11-13T10:39:17,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:17,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:17,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. after waiting 0 ms 2024-11-13T10:39:17,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:17,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:17,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 667b2b6bc191e21181054b3ac4dbe50b: Waiting for close lock at 1731494357529Disabling compacts and flushes for region at 1731494357529Disabling writes for close at 1731494357529Writing region close event to WAL at 1731494357529Closed at 1731494357529 2024-11-13T10:39:17,530 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T10:39:17,530 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731494357530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731494357530"}]},"ts":"1731494357530"} 2024-11-13T10:39:17,532 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T10:39:17,533 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T10:39:17,534 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494357533"}]},"ts":"1731494357533"} 2024-11-13T10:39:17,535 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T10:39:17,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=667b2b6bc191e21181054b3ac4dbe50b, ASSIGN}] 2024-11-13T10:39:17,538 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=667b2b6bc191e21181054b3ac4dbe50b, ASSIGN 2024-11-13T10:39:17,538 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=667b2b6bc191e21181054b3ac4dbe50b, ASSIGN; state=OFFLINE, location=770665a7984d,39101,1731494292956; forceNewPlan=false, retain=false 2024-11-13T10:39:17,689 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=667b2b6bc191e21181054b3ac4dbe50b, regionState=OPENING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:39:17,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=667b2b6bc191e21181054b3ac4dbe50b, ASSIGN because future has completed 2024-11-13T10:39:17,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:39:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-13T10:39:17,846 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:17,846 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:39:17,847 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,847 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:39:17,847 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,847 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,848 INFO [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,849 INFO [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 667b2b6bc191e21181054b3ac4dbe50b columnFamilyName A 2024-11-13T10:39:17,849 DEBUG [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:17,850 INFO [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] regionserver.HStore(327): Store=667b2b6bc191e21181054b3ac4dbe50b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:39:17,850 INFO [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,851 INFO [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 667b2b6bc191e21181054b3ac4dbe50b columnFamilyName B 2024-11-13T10:39:17,851 DEBUG [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:17,851 INFO [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] regionserver.HStore(327): Store=667b2b6bc191e21181054b3ac4dbe50b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:39:17,851 INFO [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,852 INFO [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 667b2b6bc191e21181054b3ac4dbe50b columnFamilyName C 2024-11-13T10:39:17,852 DEBUG [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:17,852 INFO [StoreOpener-667b2b6bc191e21181054b3ac4dbe50b-1 {}] regionserver.HStore(327): Store=667b2b6bc191e21181054b3ac4dbe50b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:39:17,852 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,853 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,853 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,854 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,854 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,854 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:39:17,855 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,857 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T10:39:17,857 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 667b2b6bc191e21181054b3ac4dbe50b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70227562, jitterRate=0.0464722216129303}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:39:17,858 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:17,858 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 667b2b6bc191e21181054b3ac4dbe50b: Running coprocessor pre-open hook at 1731494357847Writing region info on filesystem at 1731494357847Initializing all the Stores at 1731494357847Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494357847Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494357848 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494357848Cleaning up temporary data from old regions at 1731494357854 (+6 ms)Running coprocessor post-open hooks at 1731494357858 (+4 ms)Region opened successfully at 1731494357858 2024-11-13T10:39:17,859 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., pid=61, masterSystemTime=1731494357843 2024-11-13T10:39:17,860 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:17,860 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:17,861 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=667b2b6bc191e21181054b3ac4dbe50b, regionState=OPEN, openSeqNum=2, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:39:17,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:39:17,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T10:39:17,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 in 172 msec 2024-11-13T10:39:17,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-13T10:39:17,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=667b2b6bc191e21181054b3ac4dbe50b, ASSIGN in 328 msec 2024-11-13T10:39:17,866 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T10:39:17,866 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494357866"}]},"ts":"1731494357866"} 2024-11-13T10:39:17,868 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T10:39:17,868 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T10:39:17,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 802 msec 2024-11-13T10:39:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-13T10:39:18,207 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:18,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333e91c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,210 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2624fe81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,211 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4904bd6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,216 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T10:39:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 770665a7984d,44523,1731494292131 2024-11-13T10:39:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78041599 2024-11-13T10:39:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T10:39:18,217 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T10:39:18,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25146acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,218 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,219 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@701f7ad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,220 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5ccb97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,222 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6fc871, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,222 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,223 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cfc979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,225 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dd67fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,226 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fcff2b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,228 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d4ceeab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,230 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9c625e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,231 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@296f2de0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,233 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26172aeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,237 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d7e0c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,239 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c196787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,241 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@296895c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,243 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6de95e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,245 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5201384e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,247 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d19df36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,248 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74e3ce9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,250 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42228ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:18,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:18,251 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd7232f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:18,253 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be75eb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:18,254 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,254 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,254 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,255 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,256 DEBUG [Thread-1153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,257 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,257 DEBUG [Thread-1153 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,257 DEBUG [Thread-1153 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,258 DEBUG [Thread-1157 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:18,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-13T10:39:18,261 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T10:39:18,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:39:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:39:18,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:39:18,266 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:18,267 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,267 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:18,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:39:18,268 DEBUG [Thread-1161 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,268 DEBUG [Thread-1161 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,268 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:39:18,269 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,270 DEBUG [Thread-1166 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:18,270 DEBUG [Thread-1166 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:18,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:39:18,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:18,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:39:18,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:39:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:39:18,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:39:18,290 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:18,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:39:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:18,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/c1849d49c70746acae1bad566b9d4d3e is 50, key is test_row_0/A:col10/1731494358279/Put/seqid=0 2024-11-13T10:39:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T10:39:18,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742074_1250 (size=14341) 2024-11-13T10:39:18,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/c1849d49c70746acae1bad566b9d4d3e 2024-11-13T10:39:18,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494418369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494418373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494418376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494418379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494418389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T10:39:18,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:18,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:18,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:18,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/0a4e78a7deed401d96b970d233b2697e is 50, key is test_row_0/B:col10/1731494358279/Put/seqid=0 2024-11-13T10:39:18,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742075_1251 (size=12001) 2024-11-13T10:39:18,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/0a4e78a7deed401d96b970d233b2697e 2024-11-13T10:39:18,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494418486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494418487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494418487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494418497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494418501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/0df3be0a300d4d68b2dd4b2af7902de3 is 50, key is test_row_0/C:col10/1731494358279/Put/seqid=0 2024-11-13T10:39:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T10:39:18,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T10:39:18,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:18,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:18,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:18,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742076_1252 (size=12001) 2024-11-13T10:39:18,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494418700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494418700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494418701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494418709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494418712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:18,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T10:39:18,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:18,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:18,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:18,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T10:39:18,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T10:39:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:18,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:19,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/0df3be0a300d4d68b2dd4b2af7902de3 2024-11-13T10:39:19,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494419013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494419017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494419017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494419020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/c1849d49c70746acae1bad566b9d4d3e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c1849d49c70746acae1bad566b9d4d3e 2024-11-13T10:39:19,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494419030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c1849d49c70746acae1bad566b9d4d3e, entries=200, sequenceid=13, filesize=14.0 K 2024-11-13T10:39:19,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/0a4e78a7deed401d96b970d233b2697e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/0a4e78a7deed401d96b970d233b2697e 2024-11-13T10:39:19,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/0a4e78a7deed401d96b970d233b2697e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T10:39:19,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/0df3be0a300d4d68b2dd4b2af7902de3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/0df3be0a300d4d68b2dd4b2af7902de3 2024-11-13T10:39:19,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/0df3be0a300d4d68b2dd4b2af7902de3, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T10:39:19,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 667b2b6bc191e21181054b3ac4dbe50b in 764ms, sequenceid=13, compaction requested=false 2024-11-13T10:39:19,054 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T10:39:19,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:19,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-13T10:39:19,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:19,059 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:39:19,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8e9f04a58a6841589e505db7869d8ed9 is 50, key is test_row_0/A:col10/1731494358377/Put/seqid=0 2024-11-13T10:39:19,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742077_1253 (size=12001) 2024-11-13T10:39:19,088 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8e9f04a58a6841589e505db7869d8ed9 2024-11-13T10:39:19,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/5fe84387b8554e8bb33b75d72b313cf8 is 50, key is test_row_0/B:col10/1731494358377/Put/seqid=0 2024-11-13T10:39:19,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742078_1254 (size=12001) 2024-11-13T10:39:19,149 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/5fe84387b8554e8bb33b75d72b313cf8 2024-11-13T10:39:19,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/d665ee930cac4754923e818ff050cb26 is 50, key is test_row_0/C:col10/1731494358377/Put/seqid=0 2024-11-13T10:39:19,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742079_1255 (size=12001) 2024-11-13T10:39:19,205 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/d665ee930cac4754923e818ff050cb26 2024-11-13T10:39:19,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8e9f04a58a6841589e505db7869d8ed9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8e9f04a58a6841589e505db7869d8ed9 2024-11-13T10:39:19,215 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8e9f04a58a6841589e505db7869d8ed9, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T10:39:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/5fe84387b8554e8bb33b75d72b313cf8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/5fe84387b8554e8bb33b75d72b313cf8 2024-11-13T10:39:19,227 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/5fe84387b8554e8bb33b75d72b313cf8, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T10:39:19,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/d665ee930cac4754923e818ff050cb26 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d665ee930cac4754923e818ff050cb26 2024-11-13T10:39:19,238 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d665ee930cac4754923e818ff050cb26, entries=150, sequenceid=38, filesize=11.7 K 2024-11-13T10:39:19,240 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 667b2b6bc191e21181054b3ac4dbe50b in 181ms, sequenceid=38, compaction requested=false 2024-11-13T10:39:19,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:19,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:19,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-13T10:39:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-13T10:39:19,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T10:39:19,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 975 msec 2024-11-13T10:39:19,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 986 msec 2024-11-13T10:39:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-13T10:39:19,397 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:19,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-13T10:39:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T10:39:19,401 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:19,402 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:19,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T10:39:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:19,539 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:19,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:39:19,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/ca876fd310f94d55a490d433fe6df7f3 is 50, key is test_row_1/A:col10/1731494359533/Put/seqid=0 2024-11-13T10:39:19,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T10:39:19,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:19,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:19,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:19,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:19,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:19,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742080_1256 (size=9657) 2024-11-13T10:39:19,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/ca876fd310f94d55a490d433fe6df7f3 2024-11-13T10:39:19,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/7601e1403bcd477088d813eac083a3d9 is 50, key is test_row_1/B:col10/1731494359533/Put/seqid=0 2024-11-13T10:39:19,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494419576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494419576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494419581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494419582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494419582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742081_1257 (size=9657) 2024-11-13T10:39:19,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494419698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494419699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494419699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494419699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494419699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T10:39:19,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:19,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:19,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:19,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:19,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T10:39:19,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T10:39:19,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:19,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:19,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:19,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:19,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:19,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494419906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494419907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494419907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494419907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494419907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/7601e1403bcd477088d813eac083a3d9 2024-11-13T10:39:20,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T10:39:20,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:20,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:20,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:20,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:20,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:20,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/f01e208eabff4398bd5cd6920565c2b4 is 50, key is test_row_1/C:col10/1731494359533/Put/seqid=0 2024-11-13T10:39:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T10:39:20,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742082_1258 (size=9657) 2024-11-13T10:39:20,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/f01e208eabff4398bd5cd6920565c2b4 2024-11-13T10:39:20,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/ca876fd310f94d55a490d433fe6df7f3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ca876fd310f94d55a490d433fe6df7f3 2024-11-13T10:39:20,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ca876fd310f94d55a490d433fe6df7f3, entries=100, sequenceid=50, filesize=9.4 K 2024-11-13T10:39:20,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/7601e1403bcd477088d813eac083a3d9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7601e1403bcd477088d813eac083a3d9 2024-11-13T10:39:20,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7601e1403bcd477088d813eac083a3d9, entries=100, sequenceid=50, filesize=9.4 K 2024-11-13T10:39:20,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/f01e208eabff4398bd5cd6920565c2b4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f01e208eabff4398bd5cd6920565c2b4 2024-11-13T10:39:20,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f01e208eabff4398bd5cd6920565c2b4, entries=100, sequenceid=50, filesize=9.4 K 2024-11-13T10:39:20,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 667b2b6bc191e21181054b3ac4dbe50b in 628ms, sequenceid=50, compaction requested=true 2024-11-13T10:39:20,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:20,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:20,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:20,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:20,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:20,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:20,168 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:20,168 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:20,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:20,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T10:39:20,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:20,171 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:39:20,173 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:20,173 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/A is initiating minor compaction (all files) 2024-11-13T10:39:20,173 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/A in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:20,174 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c1849d49c70746acae1bad566b9d4d3e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8e9f04a58a6841589e505db7869d8ed9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ca876fd310f94d55a490d433fe6df7f3] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=35.2 K 2024-11-13T10:39:20,175 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:20,175 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/B is initiating minor compaction (all files) 2024-11-13T10:39:20,175 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1849d49c70746acae1bad566b9d4d3e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731494358279 2024-11-13T10:39:20,175 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/B in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:20,175 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/0a4e78a7deed401d96b970d233b2697e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/5fe84387b8554e8bb33b75d72b313cf8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7601e1403bcd477088d813eac083a3d9] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=32.9 K 2024-11-13T10:39:20,176 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e9f04a58a6841589e505db7869d8ed9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731494358336 2024-11-13T10:39:20,176 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a4e78a7deed401d96b970d233b2697e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731494358279 2024-11-13T10:39:20,176 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca876fd310f94d55a490d433fe6df7f3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494359533 2024-11-13T10:39:20,176 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fe84387b8554e8bb33b75d72b313cf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731494358336 2024-11-13T10:39:20,178 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7601e1403bcd477088d813eac083a3d9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494359533 2024-11-13T10:39:20,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/2463b907782f4a649745afb178d7b29b is 50, key is test_row_0/A:col10/1731494359575/Put/seqid=0 2024-11-13T10:39:20,214 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#A#compaction#214 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:20,214 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/4d3ba45b352d47be8273413f99537957 is 50, key is test_row_0/A:col10/1731494358377/Put/seqid=0 2024-11-13T10:39:20,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742083_1259 (size=12001) 2024-11-13T10:39:20,221 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/2463b907782f4a649745afb178d7b29b 2024-11-13T10:39:20,223 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#B#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:20,223 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/daf928495ba548d1ba7aff9ebf939c18 is 50, key is test_row_0/B:col10/1731494358377/Put/seqid=0 2024-11-13T10:39:20,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:20,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494420226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494420227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494420227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742084_1260 (size=12104) 2024-11-13T10:39:20,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494420232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494420236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,242 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/4d3ba45b352d47be8273413f99537957 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/4d3ba45b352d47be8273413f99537957 2024-11-13T10:39:20,248 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/A of 667b2b6bc191e21181054b3ac4dbe50b into 4d3ba45b352d47be8273413f99537957(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:20,248 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:20,248 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/A, priority=13, startTime=1731494360168; duration=0sec 2024-11-13T10:39:20,248 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:20,248 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:A 2024-11-13T10:39:20,249 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:20,250 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:20,251 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/C is initiating minor compaction (all files) 2024-11-13T10:39:20,251 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/C in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:20,251 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/0df3be0a300d4d68b2dd4b2af7902de3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d665ee930cac4754923e818ff050cb26, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f01e208eabff4398bd5cd6920565c2b4] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=32.9 K 2024-11-13T10:39:20,251 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0df3be0a300d4d68b2dd4b2af7902de3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731494358279 2024-11-13T10:39:20,252 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d665ee930cac4754923e818ff050cb26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731494358336 2024-11-13T10:39:20,252 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f01e208eabff4398bd5cd6920565c2b4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494359533 2024-11-13T10:39:20,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/a87ef49f520147e492812960e990ce61 is 50, key is test_row_0/B:col10/1731494359575/Put/seqid=0 2024-11-13T10:39:20,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742085_1261 (size=12104) 2024-11-13T10:39:20,299 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#C#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:20,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742086_1262 (size=12001) 2024-11-13T10:39:20,299 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/d477f65f377448198608504230b9d025 is 50, key is test_row_0/C:col10/1731494358377/Put/seqid=0 2024-11-13T10:39:20,300 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/a87ef49f520147e492812960e990ce61 2024-11-13T10:39:20,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/e4f399ddde284432a28c2156dac60aa3 is 50, key is test_row_0/C:col10/1731494359575/Put/seqid=0 2024-11-13T10:39:20,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494420337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494420337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742088_1264 (size=12001) 2024-11-13T10:39:20,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494420349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494420350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494420350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742087_1263 (size=12104) 2024-11-13T10:39:20,374 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/d477f65f377448198608504230b9d025 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d477f65f377448198608504230b9d025 2024-11-13T10:39:20,381 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/C of 667b2b6bc191e21181054b3ac4dbe50b into d477f65f377448198608504230b9d025(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:20,381 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:20,381 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/C, priority=13, startTime=1731494360168; duration=0sec 2024-11-13T10:39:20,381 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:20,381 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:C 2024-11-13T10:39:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T10:39:20,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494420547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494420547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494420558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494420558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494420558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,684 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/daf928495ba548d1ba7aff9ebf939c18 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/daf928495ba548d1ba7aff9ebf939c18 2024-11-13T10:39:20,690 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/B of 667b2b6bc191e21181054b3ac4dbe50b into daf928495ba548d1ba7aff9ebf939c18(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:20,691 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:20,691 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/B, priority=13, startTime=1731494360168; duration=0sec 2024-11-13T10:39:20,691 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:20,691 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:B 2024-11-13T10:39:20,747 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/e4f399ddde284432a28c2156dac60aa3 2024-11-13T10:39:20,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/2463b907782f4a649745afb178d7b29b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/2463b907782f4a649745afb178d7b29b 2024-11-13T10:39:20,759 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/2463b907782f4a649745afb178d7b29b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-13T10:39:20,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/a87ef49f520147e492812960e990ce61 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a87ef49f520147e492812960e990ce61 2024-11-13T10:39:20,764 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a87ef49f520147e492812960e990ce61, entries=150, sequenceid=76, filesize=11.7 K 2024-11-13T10:39:20,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/e4f399ddde284432a28c2156dac60aa3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e4f399ddde284432a28c2156dac60aa3 2024-11-13T10:39:20,771 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e4f399ddde284432a28c2156dac60aa3, entries=150, sequenceid=76, filesize=11.7 K 2024-11-13T10:39:20,771 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 667b2b6bc191e21181054b3ac4dbe50b in 600ms, sequenceid=76, compaction requested=false 2024-11-13T10:39:20,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:20,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:20,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-13T10:39:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-13T10:39:20,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-13T10:39:20,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3710 sec 2024-11-13T10:39:20,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.3820 sec 2024-11-13T10:39:20,870 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:20,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T10:39:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:20,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/d088021bb65649cbb84744f73878ffc6 is 50, key is test_row_0/A:col10/1731494360225/Put/seqid=0 2024-11-13T10:39:20,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742089_1265 (size=12001) 2024-11-13T10:39:20,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494420977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494420977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494420978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494420982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:20,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494420984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494421086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494421086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494421086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494421097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494421097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494421296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494421296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494421296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494421307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494421307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/d088021bb65649cbb84744f73878ffc6 2024-11-13T10:39:21,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/7cc5c9ccad7b4c61a755f2213922636b is 50, key is test_row_0/B:col10/1731494360225/Put/seqid=0 2024-11-13T10:39:21,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742090_1266 (size=12001) 2024-11-13T10:39:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T10:39:21,547 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:21,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-13T10:39:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T10:39:21,550 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:21,551 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:21,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:21,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494421607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494421607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494421607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494421616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494421616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T10:39:21,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T10:39:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:21,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:21,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/7cc5c9ccad7b4c61a755f2213922636b 2024-11-13T10:39:21,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/b84185b8b96e463a8d4bf9fc55092b78 is 50, key is test_row_0/C:col10/1731494360225/Put/seqid=0 2024-11-13T10:39:21,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742091_1267 (size=12001) 2024-11-13T10:39:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T10:39:21,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T10:39:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:21,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:22,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T10:39:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:22,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:22,035 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T10:39:22,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494422117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:22,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:22,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:22,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:22,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494422127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:22,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494422130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:22,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494422130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:22,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494422130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:22,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/b84185b8b96e463a8d4bf9fc55092b78 2024-11-13T10:39:22,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/d088021bb65649cbb84744f73878ffc6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d088021bb65649cbb84744f73878ffc6 2024-11-13T10:39:22,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d088021bb65649cbb84744f73878ffc6, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T10:39:22,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/7cc5c9ccad7b4c61a755f2213922636b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7cc5c9ccad7b4c61a755f2213922636b 2024-11-13T10:39:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T10:39:22,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7cc5c9ccad7b4c61a755f2213922636b, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T10:39:22,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/b84185b8b96e463a8d4bf9fc55092b78 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b84185b8b96e463a8d4bf9fc55092b78 2024-11-13T10:39:22,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T10:39:22,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:22,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:22,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:22,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:22,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:22,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b84185b8b96e463a8d4bf9fc55092b78, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T10:39:22,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 667b2b6bc191e21181054b3ac4dbe50b in 1321ms, sequenceid=91, compaction requested=true 2024-11-13T10:39:22,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:22,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:22,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:22,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:22,191 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:22,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:22,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:22,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:39:22,192 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:22,192 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:22,192 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/A is initiating minor compaction (all files) 2024-11-13T10:39:22,193 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/A in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:22,193 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/4d3ba45b352d47be8273413f99537957, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/2463b907782f4a649745afb178d7b29b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d088021bb65649cbb84744f73878ffc6] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=35.3 K 2024-11-13T10:39:22,197 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d3ba45b352d47be8273413f99537957, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494358377 2024-11-13T10:39:22,198 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2463b907782f4a649745afb178d7b29b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731494359570 2024-11-13T10:39:22,198 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting d088021bb65649cbb84744f73878ffc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731494360218 2024-11-13T10:39:22,198 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:22,198 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/B is initiating minor compaction (all files) 2024-11-13T10:39:22,198 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/B in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:22,198 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/daf928495ba548d1ba7aff9ebf939c18, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a87ef49f520147e492812960e990ce61, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7cc5c9ccad7b4c61a755f2213922636b] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=35.3 K 2024-11-13T10:39:22,199 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting daf928495ba548d1ba7aff9ebf939c18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494358377 2024-11-13T10:39:22,199 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a87ef49f520147e492812960e990ce61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731494359570 2024-11-13T10:39:22,202 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc5c9ccad7b4c61a755f2213922636b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731494360218 2024-11-13T10:39:22,215 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:22,216 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/7b7fca4342214538a5e6a676a5717a00 is 50, key is test_row_0/B:col10/1731494360225/Put/seqid=0 2024-11-13T10:39:22,226 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:22,226 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8da1e45a16ff411ea9b23ba4c8aa7349 is 50, key is test_row_0/A:col10/1731494360225/Put/seqid=0 2024-11-13T10:39:22,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742092_1268 (size=12207) 2024-11-13T10:39:22,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742093_1269 (size=12207) 2024-11-13T10:39:22,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T10:39:22,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:22,338 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:39:22,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/d2ca92aa034248b79d7a31c645949ea5 is 50, key is test_row_0/A:col10/1731494360981/Put/seqid=0 2024-11-13T10:39:22,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742094_1270 (size=12001) 2024-11-13T10:39:22,351 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/d2ca92aa034248b79d7a31c645949ea5 2024-11-13T10:39:22,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/c8b16543fc30466baa4635d66544782f is 50, key is test_row_0/B:col10/1731494360981/Put/seqid=0 2024-11-13T10:39:22,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742095_1271 (size=12001) 2024-11-13T10:39:22,636 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/7b7fca4342214538a5e6a676a5717a00 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7b7fca4342214538a5e6a676a5717a00 2024-11-13T10:39:22,642 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/B of 667b2b6bc191e21181054b3ac4dbe50b into 7b7fca4342214538a5e6a676a5717a00(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:22,642 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:22,642 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/B, priority=13, startTime=1731494362191; duration=0sec 2024-11-13T10:39:22,642 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:22,642 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:B 2024-11-13T10:39:22,642 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:22,643 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:22,643 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/C is initiating minor compaction (all files) 2024-11-13T10:39:22,643 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/C in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:22,643 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d477f65f377448198608504230b9d025, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e4f399ddde284432a28c2156dac60aa3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b84185b8b96e463a8d4bf9fc55092b78] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=35.3 K 2024-11-13T10:39:22,643 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d477f65f377448198608504230b9d025, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494358377 2024-11-13T10:39:22,644 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f399ddde284432a28c2156dac60aa3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731494359570 2024-11-13T10:39:22,644 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b84185b8b96e463a8d4bf9fc55092b78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731494360218 2024-11-13T10:39:22,652 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:22,652 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/659cc70e348c40fab13278c58039c3d2 is 50, key is test_row_0/C:col10/1731494360225/Put/seqid=0 2024-11-13T10:39:22,662 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8da1e45a16ff411ea9b23ba4c8aa7349 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8da1e45a16ff411ea9b23ba4c8aa7349 2024-11-13T10:39:22,667 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/A of 667b2b6bc191e21181054b3ac4dbe50b into 8da1e45a16ff411ea9b23ba4c8aa7349(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:22,667 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:22,667 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/A, priority=13, startTime=1731494362191; duration=0sec 2024-11-13T10:39:22,667 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:22,667 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:A 2024-11-13T10:39:22,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742096_1272 (size=12207) 2024-11-13T10:39:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T10:39:22,766 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/c8b16543fc30466baa4635d66544782f 2024-11-13T10:39:22,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/fc6ebd64c5f24768bd6141e381e6cd69 is 50, key is test_row_0/C:col10/1731494360981/Put/seqid=0 2024-11-13T10:39:22,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742097_1273 (size=12001) 2024-11-13T10:39:22,794 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/fc6ebd64c5f24768bd6141e381e6cd69 2024-11-13T10:39:22,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/d2ca92aa034248b79d7a31c645949ea5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d2ca92aa034248b79d7a31c645949ea5 2024-11-13T10:39:22,809 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d2ca92aa034248b79d7a31c645949ea5, entries=150, sequenceid=116, filesize=11.7 K 2024-11-13T10:39:22,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/c8b16543fc30466baa4635d66544782f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/c8b16543fc30466baa4635d66544782f 2024-11-13T10:39:22,816 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/c8b16543fc30466baa4635d66544782f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-13T10:39:22,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/fc6ebd64c5f24768bd6141e381e6cd69 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fc6ebd64c5f24768bd6141e381e6cd69 2024-11-13T10:39:22,823 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fc6ebd64c5f24768bd6141e381e6cd69, entries=150, sequenceid=116, filesize=11.7 K 2024-11-13T10:39:22,825 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 667b2b6bc191e21181054b3ac4dbe50b in 487ms, sequenceid=116, compaction requested=false 2024-11-13T10:39:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-13T10:39:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-13T10:39:22,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-13T10:39:22,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2760 sec 2024-11-13T10:39:22,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.2830 sec 2024-11-13T10:39:23,080 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/659cc70e348c40fab13278c58039c3d2 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/659cc70e348c40fab13278c58039c3d2 2024-11-13T10:39:23,085 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/C of 667b2b6bc191e21181054b3ac4dbe50b into 659cc70e348c40fab13278c58039c3d2(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:23,085 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:23,085 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/C, priority=13, startTime=1731494362191; duration=0sec 2024-11-13T10:39:23,085 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:23,085 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:C 2024-11-13T10:39:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:23,142 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:23,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:39:23,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8b6b9657a68e43dbb4afdb3093c2ad91 is 50, key is test_row_0/A:col10/1731494363140/Put/seqid=0 2024-11-13T10:39:23,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742098_1274 (size=12101) 2024-11-13T10:39:23,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494423176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494423177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494423178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494423179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494423180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494423288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494423288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494423288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494423288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494423298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494423497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494423497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494423498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494423498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494423507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8b6b9657a68e43dbb4afdb3093c2ad91 2024-11-13T10:39:23,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/114c49c049a34b5a8e37effd3185faf4 is 50, key is test_row_0/B:col10/1731494363140/Put/seqid=0 2024-11-13T10:39:23,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742099_1275 (size=12101) 2024-11-13T10:39:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T10:39:23,697 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:23,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-13T10:39:23,701 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:23,702 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:23,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T10:39:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T10:39:23,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494423806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494423807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494423807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494423807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494423827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:23,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T10:39:23,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:23,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:23,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:23,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:23,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:23,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:24,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T10:39:24,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:24,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:24,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:24,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:24,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T10:39:24,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/114c49c049a34b5a8e37effd3185faf4 2024-11-13T10:39:24,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/b0962f9aa8d14883873f73519d980c08 is 50, key is test_row_0/C:col10/1731494363140/Put/seqid=0 2024-11-13T10:39:24,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742100_1276 (size=12101) 2024-11-13T10:39:24,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/b0962f9aa8d14883873f73519d980c08 2024-11-13T10:39:24,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8b6b9657a68e43dbb4afdb3093c2ad91 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8b6b9657a68e43dbb4afdb3093c2ad91 2024-11-13T10:39:24,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8b6b9657a68e43dbb4afdb3093c2ad91, entries=150, sequenceid=130, filesize=11.8 K 2024-11-13T10:39:24,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/114c49c049a34b5a8e37effd3185faf4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/114c49c049a34b5a8e37effd3185faf4 2024-11-13T10:39:24,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/114c49c049a34b5a8e37effd3185faf4, entries=150, sequenceid=130, filesize=11.8 K 2024-11-13T10:39:24,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/b0962f9aa8d14883873f73519d980c08 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b0962f9aa8d14883873f73519d980c08 2024-11-13T10:39:24,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b0962f9aa8d14883873f73519d980c08, entries=150, sequenceid=130, filesize=11.8 K 2024-11-13T10:39:24,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 667b2b6bc191e21181054b3ac4dbe50b in 938ms, sequenceid=130, compaction requested=true 2024-11-13T10:39:24,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:24,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:24,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:24,080 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:24,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:24,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:24,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:24,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:24,080 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:24,081 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:24,081 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/A is initiating minor compaction (all files) 2024-11-13T10:39:24,081 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/A in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:24,082 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:24,082 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8da1e45a16ff411ea9b23ba4c8aa7349, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d2ca92aa034248b79d7a31c645949ea5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8b6b9657a68e43dbb4afdb3093c2ad91] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=35.5 K 2024-11-13T10:39:24,082 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/B is initiating minor compaction (all files) 2024-11-13T10:39:24,082 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/B in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:24,082 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7b7fca4342214538a5e6a676a5717a00, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/c8b16543fc30466baa4635d66544782f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/114c49c049a34b5a8e37effd3185faf4] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=35.5 K 2024-11-13T10:39:24,082 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8da1e45a16ff411ea9b23ba4c8aa7349, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731494360218 2024-11-13T10:39:24,082 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7fca4342214538a5e6a676a5717a00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731494360218 2024-11-13T10:39:24,082 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2ca92aa034248b79d7a31c645949ea5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731494360975 2024-11-13T10:39:24,083 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting c8b16543fc30466baa4635d66544782f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731494360975 2024-11-13T10:39:24,083 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b6b9657a68e43dbb4afdb3093c2ad91, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494363137 2024-11-13T10:39:24,083 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 114c49c049a34b5a8e37effd3185faf4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494363137 2024-11-13T10:39:24,099 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:24,100 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/d2c39340c1394f5a9effa91788f41845 is 50, key is test_row_0/B:col10/1731494363140/Put/seqid=0 2024-11-13T10:39:24,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742101_1277 (size=12409) 2024-11-13T10:39:24,106 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#A#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:24,107 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/d84330e01f5a4cdfaff29755b6e9aa27 is 50, key is test_row_0/A:col10/1731494363140/Put/seqid=0 2024-11-13T10:39:24,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742102_1278 (size=12409) 2024-11-13T10:39:24,110 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/d2c39340c1394f5a9effa91788f41845 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/d2c39340c1394f5a9effa91788f41845 2024-11-13T10:39:24,117 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/d84330e01f5a4cdfaff29755b6e9aa27 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d84330e01f5a4cdfaff29755b6e9aa27 2024-11-13T10:39:24,117 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/B of 667b2b6bc191e21181054b3ac4dbe50b into d2c39340c1394f5a9effa91788f41845(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:24,117 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:24,117 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/B, priority=13, startTime=1731494364080; duration=0sec 2024-11-13T10:39:24,117 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:24,117 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:B 2024-11-13T10:39:24,117 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:24,118 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:24,119 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/C is initiating minor compaction (all files) 2024-11-13T10:39:24,119 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/C in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:24,119 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/659cc70e348c40fab13278c58039c3d2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fc6ebd64c5f24768bd6141e381e6cd69, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b0962f9aa8d14883873f73519d980c08] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=35.5 K 2024-11-13T10:39:24,119 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 659cc70e348c40fab13278c58039c3d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731494360218 2024-11-13T10:39:24,120 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting fc6ebd64c5f24768bd6141e381e6cd69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731494360975 2024-11-13T10:39:24,121 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting b0962f9aa8d14883873f73519d980c08, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494363137 2024-11-13T10:39:24,122 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/A of 667b2b6bc191e21181054b3ac4dbe50b into d84330e01f5a4cdfaff29755b6e9aa27(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:24,122 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:24,122 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/A, priority=13, startTime=1731494364080; duration=0sec 2024-11-13T10:39:24,122 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:24,123 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:A 2024-11-13T10:39:24,144 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:24,145 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/04a0e162e38a4aad99df2a99eabbefef is 50, key is test_row_0/C:col10/1731494363140/Put/seqid=0 2024-11-13T10:39:24,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742103_1279 (size=12409) 2024-11-13T10:39:24,171 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/04a0e162e38a4aad99df2a99eabbefef as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/04a0e162e38a4aad99df2a99eabbefef 2024-11-13T10:39:24,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T10:39:24,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:24,176 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:39:24,179 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/C of 667b2b6bc191e21181054b3ac4dbe50b into 04a0e162e38a4aad99df2a99eabbefef(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:24,179 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:24,179 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/C, priority=13, startTime=1731494364080; duration=0sec 2024-11-13T10:39:24,180 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:24,180 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:C 2024-11-13T10:39:24,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/5092882f5af64ac296ef6d1d6b8a461b is 50, key is test_row_0/A:col10/1731494363178/Put/seqid=0 2024-11-13T10:39:24,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742104_1280 (size=12151) 2024-11-13T10:39:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:24,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T10:39:24,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494424324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494424324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494424326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494424326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494424336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494424437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494424437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494424437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494424437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,596 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/5092882f5af64ac296ef6d1d6b8a461b 2024-11-13T10:39:24,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/65efe12677ac41b3b892f3ec9e935b97 is 50, key is test_row_0/B:col10/1731494363178/Put/seqid=0 2024-11-13T10:39:24,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742105_1281 (size=12151) 2024-11-13T10:39:24,611 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/65efe12677ac41b3b892f3ec9e935b97 2024-11-13T10:39:24,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/86f18f957514409d8433d962f0e3ba50 is 50, key is test_row_0/C:col10/1731494363178/Put/seqid=0 2024-11-13T10:39:24,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494424647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494424647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494424647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494424647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:24,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:24,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:24,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742106_1282 (size=12151) 2024-11-13T10:39:24,655 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/86f18f957514409d8433d962f0e3ba50 2024-11-13T10:39:24,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/5092882f5af64ac296ef6d1d6b8a461b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/5092882f5af64ac296ef6d1d6b8a461b 2024-11-13T10:39:24,669 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/5092882f5af64ac296ef6d1d6b8a461b, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T10:39:24,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/65efe12677ac41b3b892f3ec9e935b97 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/65efe12677ac41b3b892f3ec9e935b97 2024-11-13T10:39:24,682 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/65efe12677ac41b3b892f3ec9e935b97, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T10:39:24,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/86f18f957514409d8433d962f0e3ba50 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/86f18f957514409d8433d962f0e3ba50 2024-11-13T10:39:24,689 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/86f18f957514409d8433d962f0e3ba50, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T10:39:24,690 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 667b2b6bc191e21181054b3ac4dbe50b in 514ms, sequenceid=158, compaction requested=false 2024-11-13T10:39:24,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:24,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:24,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-13T10:39:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-13T10:39:24,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-13T10:39:24,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 990 msec 2024-11-13T10:39:24,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 997 msec 2024-11-13T10:39:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T10:39:24,839 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:24,839 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-13T10:39:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T10:39:24,848 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:24,859 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:24,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T10:39:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:24,960 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:24,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T10:39:24,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/55d1c31d17be414eb9c68dd3b5471f3e is 50, key is test_row_0/A:col10/1731494364960/Put/seqid=0 2024-11-13T10:39:24,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742107_1283 (size=14541) 2024-11-13T10:39:24,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/55d1c31d17be414eb9c68dd3b5471f3e 2024-11-13T10:39:24,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/3c237cb0384c4c8f9048ba766889ff4f is 50, key is test_row_0/B:col10/1731494364960/Put/seqid=0 2024-11-13T10:39:25,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494424995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494424996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494424998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494424998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:25,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742108_1284 (size=12151) 2024-11-13T10:39:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494425106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494425107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494425107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494425107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:25,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:25,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T10:39:25,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494425317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494425318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494425318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494425318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494425347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,350 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in 667b2b6bc191e21181054b3ac4dbe50b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/3c237cb0384c4c8f9048ba766889ff4f 2024-11-13T10:39:25,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/576986e764094ed095fe9a9b21e50eda is 50, key is test_row_0/C:col10/1731494364960/Put/seqid=0 2024-11-13T10:39:25,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742109_1285 (size=12151) 2024-11-13T10:39:25,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/576986e764094ed095fe9a9b21e50eda 2024-11-13T10:39:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/55d1c31d17be414eb9c68dd3b5471f3e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/55d1c31d17be414eb9c68dd3b5471f3e 2024-11-13T10:39:25,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/55d1c31d17be414eb9c68dd3b5471f3e, entries=200, sequenceid=174, filesize=14.2 K 2024-11-13T10:39:25,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/3c237cb0384c4c8f9048ba766889ff4f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3c237cb0384c4c8f9048ba766889ff4f 2024-11-13T10:39:25,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:25,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:25,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T10:39:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3c237cb0384c4c8f9048ba766889ff4f, entries=150, sequenceid=174, filesize=11.9 K 2024-11-13T10:39:25,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/576986e764094ed095fe9a9b21e50eda as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/576986e764094ed095fe9a9b21e50eda 2024-11-13T10:39:25,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/576986e764094ed095fe9a9b21e50eda, entries=150, sequenceid=174, filesize=11.9 K 2024-11-13T10:39:25,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 667b2b6bc191e21181054b3ac4dbe50b in 527ms, sequenceid=174, compaction requested=true 2024-11-13T10:39:25,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:25,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:25,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:25,487 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:25,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:25,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:25,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:25,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:25,487 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:25,488 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:25,488 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/B is initiating minor compaction (all files) 2024-11-13T10:39:25,488 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/B in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,489 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/d2c39340c1394f5a9effa91788f41845, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/65efe12677ac41b3b892f3ec9e935b97, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3c237cb0384c4c8f9048ba766889ff4f] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=35.9 K 2024-11-13T10:39:25,489 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:25,489 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/A is initiating minor compaction (all files) 2024-11-13T10:39:25,489 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/A in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,489 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d84330e01f5a4cdfaff29755b6e9aa27, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/5092882f5af64ac296ef6d1d6b8a461b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/55d1c31d17be414eb9c68dd3b5471f3e] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=38.2 K 2024-11-13T10:39:25,490 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting d2c39340c1394f5a9effa91788f41845, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494363137 2024-11-13T10:39:25,490 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d84330e01f5a4cdfaff29755b6e9aa27, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494363137 2024-11-13T10:39:25,491 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 65efe12677ac41b3b892f3ec9e935b97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731494363176 2024-11-13T10:39:25,492 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5092882f5af64ac296ef6d1d6b8a461b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731494363176 2024-11-13T10:39:25,492 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c237cb0384c4c8f9048ba766889ff4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731494364324 2024-11-13T10:39:25,493 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55d1c31d17be414eb9c68dd3b5471f3e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731494364324 2024-11-13T10:39:25,509 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:25,510 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/69aab81bdb6749078ec44ddf124344a7 is 50, key is test_row_0/B:col10/1731494364960/Put/seqid=0 2024-11-13T10:39:25,510 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:25,511 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/313c1d749b7645f8acb566ae61573914 is 50, key is test_row_0/A:col10/1731494364960/Put/seqid=0 2024-11-13T10:39:25,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742111_1287 (size=12561) 2024-11-13T10:39:25,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742110_1286 (size=12561) 2024-11-13T10:39:25,540 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/69aab81bdb6749078ec44ddf124344a7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/69aab81bdb6749078ec44ddf124344a7 2024-11-13T10:39:25,541 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/313c1d749b7645f8acb566ae61573914 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/313c1d749b7645f8acb566ae61573914 2024-11-13T10:39:25,549 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/B of 667b2b6bc191e21181054b3ac4dbe50b into 69aab81bdb6749078ec44ddf124344a7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:25,549 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:25,549 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/A of 667b2b6bc191e21181054b3ac4dbe50b into 313c1d749b7645f8acb566ae61573914(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:25,549 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/B, priority=13, startTime=1731494365487; duration=0sec 2024-11-13T10:39:25,549 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:25,549 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/A, priority=13, startTime=1731494365487; duration=0sec 2024-11-13T10:39:25,549 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:25,549 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:B 2024-11-13T10:39:25,549 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:25,549 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:A 2024-11-13T10:39:25,549 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:25,551 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:25,551 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/C is initiating minor compaction (all files) 2024-11-13T10:39:25,551 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/C in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,551 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/04a0e162e38a4aad99df2a99eabbefef, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/86f18f957514409d8433d962f0e3ba50, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/576986e764094ed095fe9a9b21e50eda] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=35.9 K 2024-11-13T10:39:25,551 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 04a0e162e38a4aad99df2a99eabbefef, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494363137 2024-11-13T10:39:25,551 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f18f957514409d8433d962f0e3ba50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731494363176 2024-11-13T10:39:25,552 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 576986e764094ed095fe9a9b21e50eda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731494364324 2024-11-13T10:39:25,567 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#C#compaction#242 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:25,567 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/b404f130875249f2aeb94a433d1a4641 is 50, key is test_row_0/C:col10/1731494364960/Put/seqid=0 2024-11-13T10:39:25,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742112_1288 (size=12561) 2024-11-13T10:39:25,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,629 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:25,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:25,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T10:39:25,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/6e71bfc3c9314dfa922e4568f9aee4cf is 50, key is test_row_0/A:col10/1731494364996/Put/seqid=0 2024-11-13T10:39:25,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742113_1289 (size=12151) 2024-11-13T10:39:25,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494425642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494425641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494425643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494425644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494425757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494425757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494425757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494425757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:25,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:25,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:25,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494425967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494425967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494425967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494425968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:25,978 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/b404f130875249f2aeb94a433d1a4641 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b404f130875249f2aeb94a433d1a4641 2024-11-13T10:39:25,983 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/C of 667b2b6bc191e21181054b3ac4dbe50b into b404f130875249f2aeb94a433d1a4641(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:25,983 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:25,983 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/C, priority=13, startTime=1731494365487; duration=0sec 2024-11-13T10:39:25,983 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:25,983 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:C 2024-11-13T10:39:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T10:39:26,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/6e71bfc3c9314dfa922e4568f9aee4cf 2024-11-13T10:39:26,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/e3fedd3da3fb4f2a9c3bbaf9ef1d17d8 is 50, key is test_row_0/B:col10/1731494364996/Put/seqid=0 2024-11-13T10:39:26,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742114_1290 (size=12151) 2024-11-13T10:39:26,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:26,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:26,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:26,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:26,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494426277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494426277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494426277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494426277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:26,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:26,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:26,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:26,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:26,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:26,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/e3fedd3da3fb4f2a9c3bbaf9ef1d17d8 2024-11-13T10:39:26,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/fd0d65eb325f44a4a16fb283fa55a4ae is 50, key is test_row_0/C:col10/1731494364996/Put/seqid=0 2024-11-13T10:39:26,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742115_1291 (size=12151) 2024-11-13T10:39:26,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/fd0d65eb325f44a4a16fb283fa55a4ae 2024-11-13T10:39:26,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/6e71bfc3c9314dfa922e4568f9aee4cf as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6e71bfc3c9314dfa922e4568f9aee4cf 2024-11-13T10:39:26,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6e71bfc3c9314dfa922e4568f9aee4cf, entries=150, sequenceid=199, filesize=11.9 K 2024-11-13T10:39:26,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/e3fedd3da3fb4f2a9c3bbaf9ef1d17d8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e3fedd3da3fb4f2a9c3bbaf9ef1d17d8 2024-11-13T10:39:26,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e3fedd3da3fb4f2a9c3bbaf9ef1d17d8, entries=150, sequenceid=199, filesize=11.9 K 2024-11-13T10:39:26,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/fd0d65eb325f44a4a16fb283fa55a4ae as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fd0d65eb325f44a4a16fb283fa55a4ae 2024-11-13T10:39:26,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fd0d65eb325f44a4a16fb283fa55a4ae, entries=150, sequenceid=199, filesize=11.9 K 2024-11-13T10:39:26,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 667b2b6bc191e21181054b3ac4dbe50b in 918ms, sequenceid=199, compaction requested=false 2024-11-13T10:39:26,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:26,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T10:39:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:26,555 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T10:39:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/b85c4dadb7b44203908388bffb9ed40e is 50, key is test_row_0/A:col10/1731494365641/Put/seqid=0 2024-11-13T10:39:26,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742116_1292 (size=12151) 2024-11-13T10:39:26,566 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/b85c4dadb7b44203908388bffb9ed40e 2024-11-13T10:39:26,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/eff682f8777f413d971a43d793a57aec is 50, key is test_row_0/B:col10/1731494365641/Put/seqid=0 2024-11-13T10:39:26,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742117_1293 (size=12151) 2024-11-13T10:39:26,579 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/eff682f8777f413d971a43d793a57aec 2024-11-13T10:39:26,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/4126f287f4504b698262a47511fd479c is 50, key is test_row_0/C:col10/1731494365641/Put/seqid=0 2024-11-13T10:39:26,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742118_1294 (size=12151) 2024-11-13T10:39:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:26,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:26,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494426816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494426816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494426816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494426818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494426927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494426927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494426927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494426927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:26,998 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/4126f287f4504b698262a47511fd479c 2024-11-13T10:39:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T10:39:27,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/b85c4dadb7b44203908388bffb9ed40e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/b85c4dadb7b44203908388bffb9ed40e 2024-11-13T10:39:27,027 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/b85c4dadb7b44203908388bffb9ed40e, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T10:39:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/eff682f8777f413d971a43d793a57aec as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/eff682f8777f413d971a43d793a57aec 2024-11-13T10:39:27,041 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/eff682f8777f413d971a43d793a57aec, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T10:39:27,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/4126f287f4504b698262a47511fd479c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4126f287f4504b698262a47511fd479c 2024-11-13T10:39:27,050 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4126f287f4504b698262a47511fd479c, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T10:39:27,052 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 667b2b6bc191e21181054b3ac4dbe50b in 497ms, sequenceid=213, compaction requested=true 2024-11-13T10:39:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-13T10:39:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-13T10:39:27,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-13T10:39:27,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1950 sec 2024-11-13T10:39:27,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2170 sec 2024-11-13T10:39:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:27,141 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:27,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:39:27,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/ac0dd7eb656c4bce9e7f5eb8c7cb00fa is 50, key is test_row_0/A:col10/1731494367140/Put/seqid=0 2024-11-13T10:39:27,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742119_1295 (size=14541) 2024-11-13T10:39:27,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494427154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494427166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494427166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494427166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494427277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494427277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494427277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494427277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494427377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,378 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in 667b2b6bc191e21181054b3ac4dbe50b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494427487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494427488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494427488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494427488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/ac0dd7eb656c4bce9e7f5eb8c7cb00fa 2024-11-13T10:39:27,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/f3165ecb32d94cb682d4daed83c74e7f is 50, key is test_row_0/B:col10/1731494367140/Put/seqid=0 2024-11-13T10:39:27,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742120_1296 (size=12151) 2024-11-13T10:39:27,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494427797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494427797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494427797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494427798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:27,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:28,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/f3165ecb32d94cb682d4daed83c74e7f 2024-11-13T10:39:28,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/e61b6a14366d46319aaa43360db3efa7 is 50, key is test_row_0/C:col10/1731494367140/Put/seqid=0 2024-11-13T10:39:28,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742121_1297 (size=12151) 2024-11-13T10:39:28,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494428307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:28,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494428308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:28,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:28,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494428317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:28,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:28,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:28,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:28,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494428317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:28,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/e61b6a14366d46319aaa43360db3efa7 2024-11-13T10:39:28,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/ac0dd7eb656c4bce9e7f5eb8c7cb00fa as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ac0dd7eb656c4bce9e7f5eb8c7cb00fa 2024-11-13T10:39:28,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ac0dd7eb656c4bce9e7f5eb8c7cb00fa, entries=200, sequenceid=239, filesize=14.2 K 2024-11-13T10:39:28,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/f3165ecb32d94cb682d4daed83c74e7f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/f3165ecb32d94cb682d4daed83c74e7f 2024-11-13T10:39:28,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/f3165ecb32d94cb682d4daed83c74e7f, entries=150, sequenceid=239, filesize=11.9 K 2024-11-13T10:39:28,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/e61b6a14366d46319aaa43360db3efa7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e61b6a14366d46319aaa43360db3efa7 2024-11-13T10:39:28,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e61b6a14366d46319aaa43360db3efa7, entries=150, sequenceid=239, filesize=11.9 K 2024-11-13T10:39:28,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for 667b2b6bc191e21181054b3ac4dbe50b in 1367ms, sequenceid=239, compaction requested=true 2024-11-13T10:39:28,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:28,509 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:28,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:28,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:28,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:28,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:28,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:28,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:28,509 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:28,511 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:28,511 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/B is initiating minor compaction (all files) 2024-11-13T10:39:28,511 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/B in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:28,511 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/69aab81bdb6749078ec44ddf124344a7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e3fedd3da3fb4f2a9c3bbaf9ef1d17d8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/eff682f8777f413d971a43d793a57aec, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/f3165ecb32d94cb682d4daed83c74e7f] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=47.9 K 2024-11-13T10:39:28,511 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:28,511 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/A is initiating minor compaction (all files) 2024-11-13T10:39:28,511 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/A in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:28,512 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/313c1d749b7645f8acb566ae61573914, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6e71bfc3c9314dfa922e4568f9aee4cf, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/b85c4dadb7b44203908388bffb9ed40e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ac0dd7eb656c4bce9e7f5eb8c7cb00fa] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=50.2 K 2024-11-13T10:39:28,512 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 69aab81bdb6749078ec44ddf124344a7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731494364324 2024-11-13T10:39:28,512 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e3fedd3da3fb4f2a9c3bbaf9ef1d17d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731494364990 2024-11-13T10:39:28,512 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 313c1d749b7645f8acb566ae61573914, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731494364324 2024-11-13T10:39:28,513 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting eff682f8777f413d971a43d793a57aec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731494365637 2024-11-13T10:39:28,514 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e71bfc3c9314dfa922e4568f9aee4cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731494364990 2024-11-13T10:39:28,514 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting f3165ecb32d94cb682d4daed83c74e7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731494366816 2024-11-13T10:39:28,514 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b85c4dadb7b44203908388bffb9ed40e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731494365637 2024-11-13T10:39:28,515 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac0dd7eb656c4bce9e7f5eb8c7cb00fa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731494366816 2024-11-13T10:39:28,545 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:28,545 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/f7355c1c42c24f29bea05df09df8987b is 50, key is test_row_0/A:col10/1731494367140/Put/seqid=0 2024-11-13T10:39:28,551 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:28,552 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/4f1a2231154f45a5bc42d8af69ba715a is 50, key is test_row_0/B:col10/1731494367140/Put/seqid=0 2024-11-13T10:39:28,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742122_1298 (size=12697) 2024-11-13T10:39:28,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742123_1299 (size=12697) 2024-11-13T10:39:28,614 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/4f1a2231154f45a5bc42d8af69ba715a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4f1a2231154f45a5bc42d8af69ba715a 2024-11-13T10:39:28,621 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/B of 667b2b6bc191e21181054b3ac4dbe50b into 4f1a2231154f45a5bc42d8af69ba715a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:28,621 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:28,621 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/B, priority=12, startTime=1731494368509; duration=0sec 2024-11-13T10:39:28,621 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:28,621 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:B 2024-11-13T10:39:28,621 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:28,623 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:28,623 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/C is initiating minor compaction (all files) 2024-11-13T10:39:28,623 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/C in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:28,623 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b404f130875249f2aeb94a433d1a4641, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fd0d65eb325f44a4a16fb283fa55a4ae, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4126f287f4504b698262a47511fd479c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e61b6a14366d46319aaa43360db3efa7] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=47.9 K 2024-11-13T10:39:28,625 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting b404f130875249f2aeb94a433d1a4641, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731494364324 2024-11-13T10:39:28,625 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0d65eb325f44a4a16fb283fa55a4ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731494364990 2024-11-13T10:39:28,626 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4126f287f4504b698262a47511fd479c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731494365637 2024-11-13T10:39:28,626 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e61b6a14366d46319aaa43360db3efa7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731494366816 2024-11-13T10:39:28,648 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:28,648 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/fa8be71bee2b4f6db3e5457c99db19bc is 50, key is test_row_0/C:col10/1731494367140/Put/seqid=0 2024-11-13T10:39:28,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742124_1300 (size=12697) 2024-11-13T10:39:29,003 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/f7355c1c42c24f29bea05df09df8987b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/f7355c1c42c24f29bea05df09df8987b 2024-11-13T10:39:29,008 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/A of 667b2b6bc191e21181054b3ac4dbe50b into f7355c1c42c24f29bea05df09df8987b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:29,009 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:29,009 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/A, priority=12, startTime=1731494368508; duration=0sec 2024-11-13T10:39:29,009 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:29,009 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:A 2024-11-13T10:39:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T10:39:29,028 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:29,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-13T10:39:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T10:39:29,035 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:29,036 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:29,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:29,068 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/fa8be71bee2b4f6db3e5457c99db19bc as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fa8be71bee2b4f6db3e5457c99db19bc 2024-11-13T10:39:29,081 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/C of 667b2b6bc191e21181054b3ac4dbe50b into fa8be71bee2b4f6db3e5457c99db19bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:29,081 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:29,081 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/C, priority=12, startTime=1731494368509; duration=0sec 2024-11-13T10:39:29,081 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:29,081 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:C 2024-11-13T10:39:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T10:39:29,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T10:39:29,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:29,192 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T10:39:29,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/00c98d8516e344a88a4bb545979c6b4c is 50, key is test_row_0/A:col10/1731494367146/Put/seqid=0 2024-11-13T10:39:29,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742125_1301 (size=12151) 2024-11-13T10:39:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:29,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T10:39:29,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494429384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494429385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494429385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494429387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494429497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494429497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494429498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494429498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,632 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/00c98d8516e344a88a4bb545979c6b4c 2024-11-13T10:39:29,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/e40903092bf94830b509979eafe1274a is 50, key is test_row_0/B:col10/1731494367146/Put/seqid=0 2024-11-13T10:39:29,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742126_1302 (size=12151) 2024-11-13T10:39:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T10:39:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494429706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494429707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494429707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494429707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494430017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494430017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494430017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494430017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,062 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/e40903092bf94830b509979eafe1274a 2024-11-13T10:39:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/ccd3e6ed041b4118b1149b128bb8f7f6 is 50, key is test_row_0/C:col10/1731494367146/Put/seqid=0 2024-11-13T10:39:30,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742127_1303 (size=12151) 2024-11-13T10:39:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T10:39:30,491 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/ccd3e6ed041b4118b1149b128bb8f7f6 2024-11-13T10:39:30,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/00c98d8516e344a88a4bb545979c6b4c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/00c98d8516e344a88a4bb545979c6b4c 2024-11-13T10:39:30,500 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/00c98d8516e344a88a4bb545979c6b4c, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T10:39:30,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/e40903092bf94830b509979eafe1274a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e40903092bf94830b509979eafe1274a 2024-11-13T10:39:30,505 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e40903092bf94830b509979eafe1274a, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T10:39:30,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/ccd3e6ed041b4118b1149b128bb8f7f6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/ccd3e6ed041b4118b1149b128bb8f7f6 2024-11-13T10:39:30,509 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/ccd3e6ed041b4118b1149b128bb8f7f6, entries=150, sequenceid=254, filesize=11.9 K 2024-11-13T10:39:30,510 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 667b2b6bc191e21181054b3ac4dbe50b in 1318ms, sequenceid=254, compaction requested=false 2024-11-13T10:39:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-13T10:39:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-13T10:39:30,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-13T10:39:30,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4760 sec 2024-11-13T10:39:30,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4860 sec 2024-11-13T10:39:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:30,530 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:30,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:39:30,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/35200cd549354a718847e447d8c3c6ae is 50, key is test_row_0/A:col10/1731494370528/Put/seqid=0 2024-11-13T10:39:30,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742128_1304 (size=12301) 2024-11-13T10:39:30,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494430536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494430537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494430537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494430541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494430647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494430647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494430647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494430657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494430857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494430857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494430857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494430867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:30,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/35200cd549354a718847e447d8c3c6ae 2024-11-13T10:39:30,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/497d0d7f1cec42d3abbc0ef886e04b79 is 50, key is test_row_0/B:col10/1731494370528/Put/seqid=0 2024-11-13T10:39:30,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742129_1305 (size=12301) 2024-11-13T10:39:31,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494431167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494431167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494431167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494431177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T10:39:31,207 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:31,208 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-13T10:39:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T10:39:31,210 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:31,212 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:31,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T10:39:31,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/497d0d7f1cec42d3abbc0ef886e04b79 2024-11-13T10:39:31,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T10:39:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:31,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:31,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731494431397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/57a3802d9fe940ab96cccb369c2e5a08 is 50, key is test_row_0/C:col10/1731494370528/Put/seqid=0 2024-11-13T10:39:31,399 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in 667b2b6bc191e21181054b3ac4dbe50b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:31,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742130_1306 (size=12301) 2024-11-13T10:39:31,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/57a3802d9fe940ab96cccb369c2e5a08 2024-11-13T10:39:31,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/35200cd549354a718847e447d8c3c6ae as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/35200cd549354a718847e447d8c3c6ae 2024-11-13T10:39:31,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/35200cd549354a718847e447d8c3c6ae, entries=150, sequenceid=280, filesize=12.0 K 2024-11-13T10:39:31,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/497d0d7f1cec42d3abbc0ef886e04b79 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/497d0d7f1cec42d3abbc0ef886e04b79 2024-11-13T10:39:31,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/497d0d7f1cec42d3abbc0ef886e04b79, entries=150, sequenceid=280, filesize=12.0 K 2024-11-13T10:39:31,476 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-13T10:39:31,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/57a3802d9fe940ab96cccb369c2e5a08 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/57a3802d9fe940ab96cccb369c2e5a08 2024-11-13T10:39:31,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/57a3802d9fe940ab96cccb369c2e5a08, entries=150, sequenceid=280, filesize=12.0 K 2024-11-13T10:39:31,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 667b2b6bc191e21181054b3ac4dbe50b in 964ms, sequenceid=280, compaction requested=true 2024-11-13T10:39:31,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:31,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:31,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:31,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:31,494 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:31,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:31,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:31,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:31,494 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:31,495 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:31,495 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/A is initiating minor compaction (all files) 2024-11-13T10:39:31,496 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/A in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:31,496 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/f7355c1c42c24f29bea05df09df8987b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/00c98d8516e344a88a4bb545979c6b4c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/35200cd549354a718847e447d8c3c6ae] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=36.3 K 2024-11-13T10:39:31,496 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:31,496 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/B is initiating minor compaction (all files) 2024-11-13T10:39:31,496 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/B in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:31,496 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4f1a2231154f45a5bc42d8af69ba715a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e40903092bf94830b509979eafe1274a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/497d0d7f1cec42d3abbc0ef886e04b79] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=36.3 K 2024-11-13T10:39:31,496 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7355c1c42c24f29bea05df09df8987b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731494366816 2024-11-13T10:39:31,497 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f1a2231154f45a5bc42d8af69ba715a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731494366816 2024-11-13T10:39:31,498 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e40903092bf94830b509979eafe1274a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731494367146 2024-11-13T10:39:31,498 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 497d0d7f1cec42d3abbc0ef886e04b79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731494369382 2024-11-13T10:39:31,500 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00c98d8516e344a88a4bb545979c6b4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731494367146 2024-11-13T10:39:31,502 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35200cd549354a718847e447d8c3c6ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731494369382 2024-11-13T10:39:31,514 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:31,515 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/fce0cbd2c56144ad84c3bcc690081b5f is 50, key is test_row_0/B:col10/1731494370528/Put/seqid=0 2024-11-13T10:39:31,517 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:31,517 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a5b2af8bdf5e4eed8ef14e58ad65fa98 is 50, key is test_row_0/A:col10/1731494370528/Put/seqid=0 2024-11-13T10:39:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T10:39:31,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T10:39:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:31,532 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T10:39:31,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742131_1307 (size=12949) 2024-11-13T10:39:31,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742132_1308 (size=12949) 2024-11-13T10:39:31,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a87db24e67c542d182974c01cc71fa1a is 50, key is test_row_0/A:col10/1731494370540/Put/seqid=0 2024-11-13T10:39:31,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742133_1309 (size=12301) 2024-11-13T10:39:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:31,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:31,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494431721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494431722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494431722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494431730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T10:39:31,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494431837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494431837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494431837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494431847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:31,948 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/fce0cbd2c56144ad84c3bcc690081b5f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/fce0cbd2c56144ad84c3bcc690081b5f 2024-11-13T10:39:31,954 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a5b2af8bdf5e4eed8ef14e58ad65fa98 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a5b2af8bdf5e4eed8ef14e58ad65fa98 2024-11-13T10:39:31,958 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/B of 667b2b6bc191e21181054b3ac4dbe50b into fce0cbd2c56144ad84c3bcc690081b5f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:31,958 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:31,958 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/B, priority=13, startTime=1731494371494; duration=0sec 2024-11-13T10:39:31,959 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:31,959 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:B 2024-11-13T10:39:31,959 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:31,959 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/A of 667b2b6bc191e21181054b3ac4dbe50b into a5b2af8bdf5e4eed8ef14e58ad65fa98(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:31,959 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:31,959 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/A, priority=13, startTime=1731494371494; duration=0sec 2024-11-13T10:39:31,959 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:31,959 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:A 2024-11-13T10:39:31,960 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:31,960 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/C is initiating minor compaction (all files) 2024-11-13T10:39:31,960 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/C in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:31,960 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fa8be71bee2b4f6db3e5457c99db19bc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/ccd3e6ed041b4118b1149b128bb8f7f6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/57a3802d9fe940ab96cccb369c2e5a08] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=36.3 K 2024-11-13T10:39:31,960 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting fa8be71bee2b4f6db3e5457c99db19bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731494366816 2024-11-13T10:39:31,960 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd3e6ed041b4118b1149b128bb8f7f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731494367146 2024-11-13T10:39:31,961 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a3802d9fe940ab96cccb369c2e5a08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731494369382 2024-11-13T10:39:31,963 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a87db24e67c542d182974c01cc71fa1a 2024-11-13T10:39:31,969 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#C#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:31,969 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/f5099f425d004cc8b5204e05a3d8767e is 50, key is test_row_0/C:col10/1731494370528/Put/seqid=0 2024-11-13T10:39:31,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/bc0e20b5efd5496eb2c029fd2bc283bc is 50, key is test_row_0/B:col10/1731494370540/Put/seqid=0 2024-11-13T10:39:31,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742134_1310 (size=12949) 2024-11-13T10:39:31,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742135_1311 (size=12301) 2024-11-13T10:39:31,979 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/f5099f425d004cc8b5204e05a3d8767e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f5099f425d004cc8b5204e05a3d8767e 2024-11-13T10:39:31,985 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/C of 667b2b6bc191e21181054b3ac4dbe50b into f5099f425d004cc8b5204e05a3d8767e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:31,985 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:31,985 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/C, priority=13, startTime=1731494371494; duration=0sec 2024-11-13T10:39:31,986 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:31,986 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:C 2024-11-13T10:39:32,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494432046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494432046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494432046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494432057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T10:39:32,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494432357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494432357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494432357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494432367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,377 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/bc0e20b5efd5496eb2c029fd2bc283bc 2024-11-13T10:39:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/9e73357a188f4181800f1864230b271c is 50, key is test_row_0/C:col10/1731494370540/Put/seqid=0 2024-11-13T10:39:32,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742136_1312 (size=12301) 2024-11-13T10:39:32,816 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/9e73357a188f4181800f1864230b271c 2024-11-13T10:39:32,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a87db24e67c542d182974c01cc71fa1a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a87db24e67c542d182974c01cc71fa1a 2024-11-13T10:39:32,827 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a87db24e67c542d182974c01cc71fa1a, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T10:39:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/bc0e20b5efd5496eb2c029fd2bc283bc as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/bc0e20b5efd5496eb2c029fd2bc283bc 2024-11-13T10:39:32,848 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/bc0e20b5efd5496eb2c029fd2bc283bc, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T10:39:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/9e73357a188f4181800f1864230b271c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/9e73357a188f4181800f1864230b271c 2024-11-13T10:39:32,861 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/9e73357a188f4181800f1864230b271c, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T10:39:32,862 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 667b2b6bc191e21181054b3ac4dbe50b in 1330ms, sequenceid=290, compaction requested=false 2024-11-13T10:39:32,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:32,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:32,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-13T10:39:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-13T10:39:32,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-13T10:39:32,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6530 sec 2024-11-13T10:39:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:32,873 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:32,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-13T10:39:32,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6640 sec 2024-11-13T10:39:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494432880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494432881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/09c8a372e777405ba9381805f46987d2 is 50, key is test_row_0/A:col10/1731494372873/Put/seqid=0 2024-11-13T10:39:32,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494432887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494432887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:32,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742137_1313 (size=14741) 2024-11-13T10:39:32,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/09c8a372e777405ba9381805f46987d2 2024-11-13T10:39:32,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/ec81616e5cc940be945d34934fbe0865 is 50, key is test_row_0/B:col10/1731494372873/Put/seqid=0 2024-11-13T10:39:32,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742138_1314 (size=12301) 2024-11-13T10:39:32,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/ec81616e5cc940be945d34934fbe0865 2024-11-13T10:39:32,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/4548a1e3413c462a9425419c6b99c520 is 50, key is test_row_0/C:col10/1731494372873/Put/seqid=0 2024-11-13T10:39:33,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494433000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494433000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494433000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494433001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742139_1315 (size=12301) 2024-11-13T10:39:33,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/4548a1e3413c462a9425419c6b99c520 2024-11-13T10:39:33,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/09c8a372e777405ba9381805f46987d2 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/09c8a372e777405ba9381805f46987d2 2024-11-13T10:39:33,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/09c8a372e777405ba9381805f46987d2, entries=200, sequenceid=322, filesize=14.4 K 2024-11-13T10:39:33,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/ec81616e5cc940be945d34934fbe0865 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/ec81616e5cc940be945d34934fbe0865 2024-11-13T10:39:33,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/ec81616e5cc940be945d34934fbe0865, entries=150, sequenceid=322, filesize=12.0 K 2024-11-13T10:39:33,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/4548a1e3413c462a9425419c6b99c520 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4548a1e3413c462a9425419c6b99c520 2024-11-13T10:39:33,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4548a1e3413c462a9425419c6b99c520, entries=150, sequenceid=322, filesize=12.0 K 2024-11-13T10:39:33,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 667b2b6bc191e21181054b3ac4dbe50b in 232ms, sequenceid=322, compaction requested=true 2024-11-13T10:39:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:33,105 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:33,105 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:33,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:33,106 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:33,106 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:33,106 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/A is initiating minor compaction (all files) 2024-11-13T10:39:33,106 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/B is initiating minor compaction (all files) 2024-11-13T10:39:33,106 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/A in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:33,106 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/B in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:33,107 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a5b2af8bdf5e4eed8ef14e58ad65fa98, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a87db24e67c542d182974c01cc71fa1a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/09c8a372e777405ba9381805f46987d2] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=39.1 K 2024-11-13T10:39:33,107 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/fce0cbd2c56144ad84c3bcc690081b5f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/bc0e20b5efd5496eb2c029fd2bc283bc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/ec81616e5cc940be945d34934fbe0865] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=36.7 K 2024-11-13T10:39:33,107 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting fce0cbd2c56144ad84c3bcc690081b5f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731494369382 2024-11-13T10:39:33,107 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b2af8bdf5e4eed8ef14e58ad65fa98, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731494369382 2024-11-13T10:39:33,110 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting bc0e20b5efd5496eb2c029fd2bc283bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731494370534 2024-11-13T10:39:33,110 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a87db24e67c542d182974c01cc71fa1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731494370534 2024-11-13T10:39:33,121 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ec81616e5cc940be945d34934fbe0865, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731494371720 2024-11-13T10:39:33,126 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09c8a372e777405ba9381805f46987d2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731494371720 2024-11-13T10:39:33,181 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:33,182 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/e7aa0e9533dd41599624ebdf7a63850a is 50, key is test_row_0/B:col10/1731494372873/Put/seqid=0 2024-11-13T10:39:33,194 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:33,195 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/d5eacd204d83424f83fd2a7910422469 is 50, key is test_row_0/A:col10/1731494372873/Put/seqid=0 2024-11-13T10:39:33,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:33,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:33,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:39:33,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742140_1316 (size=13051) 2024-11-13T10:39:33,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742141_1317 (size=13051) 2024-11-13T10:39:33,247 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/e7aa0e9533dd41599624ebdf7a63850a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e7aa0e9533dd41599624ebdf7a63850a 2024-11-13T10:39:33,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/c52de2681cbf469388a9a0ad7fd406f3 is 50, key is test_row_0/A:col10/1731494372880/Put/seqid=0 2024-11-13T10:39:33,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742142_1318 (size=12301) 2024-11-13T10:39:33,280 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/B of 667b2b6bc191e21181054b3ac4dbe50b into e7aa0e9533dd41599624ebdf7a63850a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:33,280 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:33,280 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/B, priority=13, startTime=1731494373105; duration=0sec 2024-11-13T10:39:33,281 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:33,281 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:B 2024-11-13T10:39:33,281 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:33,293 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:33,293 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/C is initiating minor compaction (all files) 2024-11-13T10:39:33,293 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/C in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:33,293 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f5099f425d004cc8b5204e05a3d8767e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/9e73357a188f4181800f1864230b271c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4548a1e3413c462a9425419c6b99c520] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=36.7 K 2024-11-13T10:39:33,294 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting f5099f425d004cc8b5204e05a3d8767e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731494369382 2024-11-13T10:39:33,294 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e73357a188f4181800f1864230b271c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731494370534 2024-11-13T10:39:33,294 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4548a1e3413c462a9425419c6b99c520, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731494371720 2024-11-13T10:39:33,313 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:33,313 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/befdba0d34bf41f2b9462854b81f5e9a is 50, key is test_row_0/C:col10/1731494372873/Put/seqid=0 2024-11-13T10:39:33,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494433325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494433327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494433323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742143_1319 (size=13051) 2024-11-13T10:39:33,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494433331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T10:39:33,357 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:33,358 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:33,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-13T10:39:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T10:39:33,361 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:33,361 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:33,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:33,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494433437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494433440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494433440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494433447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T10:39:33,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T10:39:33,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:33,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:33,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:33,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:33,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494433647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,651 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/d5eacd204d83424f83fd2a7910422469 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d5eacd204d83424f83fd2a7910422469 2024-11-13T10:39:33,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/c52de2681cbf469388a9a0ad7fd406f3 2024-11-13T10:39:33,656 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/A of 667b2b6bc191e21181054b3ac4dbe50b into d5eacd204d83424f83fd2a7910422469(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:33,656 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:33,656 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/A, priority=13, startTime=1731494373105; duration=0sec 2024-11-13T10:39:33,656 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:33,656 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:A 2024-11-13T10:39:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494433657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494433658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494433658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/8008b1f64ed842ce99ceaad080107b5a is 50, key is test_row_0/B:col10/1731494372880/Put/seqid=0 2024-11-13T10:39:33,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T10:39:33,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:33,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:33,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:33,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:33,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742144_1320 (size=12301) 2024-11-13T10:39:33,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/8008b1f64ed842ce99ceaad080107b5a 2024-11-13T10:39:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T10:39:33,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/1fbd8ca9e44a4edbaf1d800320e265d2 is 50, key is test_row_0/C:col10/1731494372880/Put/seqid=0 2024-11-13T10:39:33,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742145_1321 (size=12301) 2024-11-13T10:39:33,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/1fbd8ca9e44a4edbaf1d800320e265d2 2024-11-13T10:39:33,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/c52de2681cbf469388a9a0ad7fd406f3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c52de2681cbf469388a9a0ad7fd406f3 2024-11-13T10:39:33,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c52de2681cbf469388a9a0ad7fd406f3, entries=150, sequenceid=333, filesize=12.0 K 2024-11-13T10:39:33,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/8008b1f64ed842ce99ceaad080107b5a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/8008b1f64ed842ce99ceaad080107b5a 2024-11-13T10:39:33,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/8008b1f64ed842ce99ceaad080107b5a, entries=150, sequenceid=333, filesize=12.0 K 2024-11-13T10:39:33,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/1fbd8ca9e44a4edbaf1d800320e265d2 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/1fbd8ca9e44a4edbaf1d800320e265d2 2024-11-13T10:39:33,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/1fbd8ca9e44a4edbaf1d800320e265d2, entries=150, sequenceid=333, filesize=12.0 K 2024-11-13T10:39:33,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 667b2b6bc191e21181054b3ac4dbe50b in 500ms, sequenceid=333, compaction requested=false 2024-11-13T10:39:33,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:33,741 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/befdba0d34bf41f2b9462854b81f5e9a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/befdba0d34bf41f2b9462854b81f5e9a 2024-11-13T10:39:33,748 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/C of 667b2b6bc191e21181054b3ac4dbe50b into befdba0d34bf41f2b9462854b81f5e9a(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:33,748 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:33,748 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/C, priority=13, startTime=1731494373105; duration=0sec 2024-11-13T10:39:33,748 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:33,748 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:C 2024-11-13T10:39:33,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T10:39:33,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:33,823 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:39:33,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/44184d0e382743b3a91fc777e51cc9a6 is 50, key is test_row_0/A:col10/1731494373328/Put/seqid=0 2024-11-13T10:39:33,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742146_1322 (size=12301) 2024-11-13T10:39:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:33,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:33,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494433972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494433973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494433973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494433974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:33,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T10:39:34,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494434076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494434086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494434087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494434087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,235 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/44184d0e382743b3a91fc777e51cc9a6 2024-11-13T10:39:34,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/4677c60dc6c14217939ea7caf39fe54e is 50, key is test_row_0/B:col10/1731494373328/Put/seqid=0 2024-11-13T10:39:34,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742147_1323 (size=12301) 2024-11-13T10:39:34,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494434286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494434297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494434297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494434301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T10:39:34,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494434596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494434607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494434607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494434617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:34,646 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/4677c60dc6c14217939ea7caf39fe54e 2024-11-13T10:39:34,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/95d1455cea3b49129dcf8054bab388ad is 50, key is test_row_0/C:col10/1731494373328/Put/seqid=0 2024-11-13T10:39:34,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742148_1324 (size=12301) 2024-11-13T10:39:35,059 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/95d1455cea3b49129dcf8054bab388ad 2024-11-13T10:39:35,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/44184d0e382743b3a91fc777e51cc9a6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/44184d0e382743b3a91fc777e51cc9a6 2024-11-13T10:39:35,081 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/44184d0e382743b3a91fc777e51cc9a6, entries=150, sequenceid=361, filesize=12.0 K 2024-11-13T10:39:35,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/4677c60dc6c14217939ea7caf39fe54e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4677c60dc6c14217939ea7caf39fe54e 2024-11-13T10:39:35,087 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4677c60dc6c14217939ea7caf39fe54e, entries=150, sequenceid=361, filesize=12.0 K 2024-11-13T10:39:35,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/95d1455cea3b49129dcf8054bab388ad as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/95d1455cea3b49129dcf8054bab388ad 2024-11-13T10:39:35,092 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/95d1455cea3b49129dcf8054bab388ad, entries=150, sequenceid=361, filesize=12.0 K 2024-11-13T10:39:35,097 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 667b2b6bc191e21181054b3ac4dbe50b in 1274ms, sequenceid=361, compaction requested=true 2024-11-13T10:39:35,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:35,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:35,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-13T10:39:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-13T10:39:35,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-13T10:39:35,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7380 sec 2024-11-13T10:39:35,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7450 sec 2024-11-13T10:39:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:35,121 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:35,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T10:39:35,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/9ea5341b4d704fb498d29e579ee508f1 is 50, key is test_row_0/A:col10/1731494373970/Put/seqid=0 2024-11-13T10:39:35,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742149_1325 (size=12301) 2024-11-13T10:39:35,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494435177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494435177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494435182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494435183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494435289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494435289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494435297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494435297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T10:39:35,510 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494435507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494435508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494435508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494435508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-13T10:39:35,522 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:35,523 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:35,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:35,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T10:39:35,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/9ea5341b4d704fb498d29e579ee508f1 2024-11-13T10:39:35,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/1976f696f672479799c28819dc9d1c8d is 50, key is test_row_0/B:col10/1731494373970/Put/seqid=0 2024-11-13T10:39:35,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742150_1326 (size=12301) 2024-11-13T10:39:35,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/1976f696f672479799c28819dc9d1c8d 2024-11-13T10:39:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T10:39:35,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/bbb390d84e7c4a9eb6a5b941133f1152 is 50, key is test_row_0/C:col10/1731494373970/Put/seqid=0 2024-11-13T10:39:35,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742151_1327 (size=12301) 2024-11-13T10:39:35,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/bbb390d84e7c4a9eb6a5b941133f1152 2024-11-13T10:39:35,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T10:39:35,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:35,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:35,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:35,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:35,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:35,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/9ea5341b4d704fb498d29e579ee508f1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/9ea5341b4d704fb498d29e579ee508f1 2024-11-13T10:39:35,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/9ea5341b4d704fb498d29e579ee508f1, entries=150, sequenceid=375, filesize=12.0 K 2024-11-13T10:39:35,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/1976f696f672479799c28819dc9d1c8d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/1976f696f672479799c28819dc9d1c8d 2024-11-13T10:39:35,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/1976f696f672479799c28819dc9d1c8d, entries=150, sequenceid=375, filesize=12.0 K 2024-11-13T10:39:35,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/bbb390d84e7c4a9eb6a5b941133f1152 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/bbb390d84e7c4a9eb6a5b941133f1152 2024-11-13T10:39:35,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/bbb390d84e7c4a9eb6a5b941133f1152, entries=150, sequenceid=375, filesize=12.0 K 2024-11-13T10:39:35,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 667b2b6bc191e21181054b3ac4dbe50b in 641ms, sequenceid=375, compaction requested=true 2024-11-13T10:39:35,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:35,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:35,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:35,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:35,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:35,762 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:35,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:35,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:39:35,762 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:35,763 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:35,763 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:35,764 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/A is initiating minor compaction (all files) 2024-11-13T10:39:35,764 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/B is initiating minor compaction (all files) 2024-11-13T10:39:35,764 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/B in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:35,764 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/A in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:35,764 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e7aa0e9533dd41599624ebdf7a63850a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/8008b1f64ed842ce99ceaad080107b5a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4677c60dc6c14217939ea7caf39fe54e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/1976f696f672479799c28819dc9d1c8d] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=48.8 K 2024-11-13T10:39:35,764 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d5eacd204d83424f83fd2a7910422469, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c52de2681cbf469388a9a0ad7fd406f3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/44184d0e382743b3a91fc777e51cc9a6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/9ea5341b4d704fb498d29e579ee508f1] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=48.8 K 2024-11-13T10:39:35,764 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e7aa0e9533dd41599624ebdf7a63850a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731494371720 2024-11-13T10:39:35,764 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5eacd204d83424f83fd2a7910422469, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731494371720 2024-11-13T10:39:35,764 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c52de2681cbf469388a9a0ad7fd406f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731494372880 2024-11-13T10:39:35,765 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8008b1f64ed842ce99ceaad080107b5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731494372880 2024-11-13T10:39:35,765 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44184d0e382743b3a91fc777e51cc9a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731494373299 2024-11-13T10:39:35,765 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4677c60dc6c14217939ea7caf39fe54e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731494373299 2024-11-13T10:39:35,765 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ea5341b4d704fb498d29e579ee508f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731494373970 2024-11-13T10:39:35,765 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1976f696f672479799c28819dc9d1c8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731494373970 2024-11-13T10:39:35,789 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#B#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:35,790 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/a2fef13d929d4fef882cd58b71434e4e is 50, key is test_row_0/B:col10/1731494373970/Put/seqid=0 2024-11-13T10:39:35,794 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#A#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:35,795 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/6318d59e5c4741c1a31a9695751cbbe6 is 50, key is test_row_0/A:col10/1731494373970/Put/seqid=0 2024-11-13T10:39:35,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742153_1329 (size=13187) 2024-11-13T10:39:35,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742152_1328 (size=13187) 2024-11-13T10:39:35,815 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/6318d59e5c4741c1a31a9695751cbbe6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6318d59e5c4741c1a31a9695751cbbe6 2024-11-13T10:39:35,821 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:35,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T10:39:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:35,821 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/A of 667b2b6bc191e21181054b3ac4dbe50b into 6318d59e5c4741c1a31a9695751cbbe6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:35,821 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:35,821 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/A, priority=12, startTime=1731494375762; duration=0sec 2024-11-13T10:39:35,822 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:35,822 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:A 2024-11-13T10:39:35,822 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:35,823 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:35,823 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/C is initiating minor compaction (all files) 2024-11-13T10:39:35,823 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/C in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:35,824 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/befdba0d34bf41f2b9462854b81f5e9a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/1fbd8ca9e44a4edbaf1d800320e265d2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/95d1455cea3b49129dcf8054bab388ad, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/bbb390d84e7c4a9eb6a5b941133f1152] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=48.8 K 2024-11-13T10:39:35,824 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting befdba0d34bf41f2b9462854b81f5e9a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731494371720 2024-11-13T10:39:35,824 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fbd8ca9e44a4edbaf1d800320e265d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731494372880 2024-11-13T10:39:35,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/6c2f460de17b42dfa7cf732356f831f8 is 50, key is test_row_0/A:col10/1731494375819/Put/seqid=0 2024-11-13T10:39:35,829 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95d1455cea3b49129dcf8054bab388ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731494373299 2024-11-13T10:39:35,830 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbb390d84e7c4a9eb6a5b941133f1152, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731494373970 2024-11-13T10:39:35,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742154_1330 (size=14741) 2024-11-13T10:39:35,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T10:39:35,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:35,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:35,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:35,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:35,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:35,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494435840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494435837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T10:39:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494435847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494435848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,858 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#C#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:35,858 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/68e44450ba594a0ea5ded9818ed05ba7 is 50, key is test_row_0/C:col10/1731494373970/Put/seqid=0 2024-11-13T10:39:35,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742155_1331 (size=13187) 2024-11-13T10:39:35,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494435961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494435961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494435961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494435962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:35,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T10:39:35,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:35,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:35,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:35,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:35,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:36,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T10:39:36,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:36,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:36,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:36,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:36,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T10:39:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:36,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494436167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494436177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494436177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494436177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,208 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/a2fef13d929d4fef882cd58b71434e4e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a2fef13d929d4fef882cd58b71434e4e 2024-11-13T10:39:36,225 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/B of 667b2b6bc191e21181054b3ac4dbe50b into a2fef13d929d4fef882cd58b71434e4e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:36,225 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:36,225 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/B, priority=12, startTime=1731494375762; duration=0sec 2024-11-13T10:39:36,225 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:36,225 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:B 2024-11-13T10:39:36,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/6c2f460de17b42dfa7cf732356f831f8 2024-11-13T10:39:36,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/6c8a3ef1cb464c7aaf87d4696dd0f53e is 50, key is test_row_0/B:col10/1731494375819/Put/seqid=0 2024-11-13T10:39:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742156_1332 (size=12301) 2024-11-13T10:39:36,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/6c8a3ef1cb464c7aaf87d4696dd0f53e 2024-11-13T10:39:36,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/e42dc0a1ca6e42bf9129e8e26ccd09e5 is 50, key is test_row_0/C:col10/1731494375819/Put/seqid=0 2024-11-13T10:39:36,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742157_1333 (size=12301) 2024-11-13T10:39:36,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/e42dc0a1ca6e42bf9129e8e26ccd09e5 2024-11-13T10:39:36,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/6c2f460de17b42dfa7cf732356f831f8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6c2f460de17b42dfa7cf732356f831f8 2024-11-13T10:39:36,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6c2f460de17b42dfa7cf732356f831f8, entries=200, sequenceid=400, filesize=14.4 K 2024-11-13T10:39:36,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/6c8a3ef1cb464c7aaf87d4696dd0f53e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6c8a3ef1cb464c7aaf87d4696dd0f53e 2024-11-13T10:39:36,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6c8a3ef1cb464c7aaf87d4696dd0f53e, entries=150, sequenceid=400, filesize=12.0 K 2024-11-13T10:39:36,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/e42dc0a1ca6e42bf9129e8e26ccd09e5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e42dc0a1ca6e42bf9129e8e26ccd09e5 2024-11-13T10:39:36,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e42dc0a1ca6e42bf9129e8e26ccd09e5, entries=150, sequenceid=400, filesize=12.0 K 2024-11-13T10:39:36,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 667b2b6bc191e21181054b3ac4dbe50b in 474ms, sequenceid=400, compaction requested=false 2024-11-13T10:39:36,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:36,298 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/68e44450ba594a0ea5ded9818ed05ba7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/68e44450ba594a0ea5ded9818ed05ba7 2024-11-13T10:39:36,305 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/C of 667b2b6bc191e21181054b3ac4dbe50b into 68e44450ba594a0ea5ded9818ed05ba7(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:36,305 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:36,305 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/C, priority=12, startTime=1731494375762; duration=0sec 2024-11-13T10:39:36,305 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:36,305 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:C 2024-11-13T10:39:36,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T10:39:36,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:36,310 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T10:39:36,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a1439489278d4b0a8bd82aa2eea13537 is 50, key is test_row_0/A:col10/1731494375845/Put/seqid=0 2024-11-13T10:39:36,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742158_1334 (size=12301) 2024-11-13T10:39:36,331 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a1439489278d4b0a8bd82aa2eea13537 2024-11-13T10:39:36,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/3e511ad753a5466abb1ab7b624ee20d6 is 50, key is test_row_0/B:col10/1731494375845/Put/seqid=0 2024-11-13T10:39:36,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742159_1335 (size=12301) 2024-11-13T10:39:36,381 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/3e511ad753a5466abb1ab7b624ee20d6 2024-11-13T10:39:36,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/2e50674f7b3c44ab9338060408b3ba87 is 50, key is test_row_0/C:col10/1731494375845/Put/seqid=0 2024-11-13T10:39:36,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742160_1336 (size=12301) 2024-11-13T10:39:36,481 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/2e50674f7b3c44ab9338060408b3ba87 2024-11-13T10:39:36,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:36,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a1439489278d4b0a8bd82aa2eea13537 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a1439489278d4b0a8bd82aa2eea13537 2024-11-13T10:39:36,542 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a1439489278d4b0a8bd82aa2eea13537, entries=150, sequenceid=415, filesize=12.0 K 2024-11-13T10:39:36,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/3e511ad753a5466abb1ab7b624ee20d6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3e511ad753a5466abb1ab7b624ee20d6 2024-11-13T10:39:36,553 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3e511ad753a5466abb1ab7b624ee20d6, entries=150, sequenceid=415, filesize=12.0 K 2024-11-13T10:39:36,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/2e50674f7b3c44ab9338060408b3ba87 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/2e50674f7b3c44ab9338060408b3ba87 2024-11-13T10:39:36,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494436549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494436549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,560 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/2e50674f7b3c44ab9338060408b3ba87, entries=150, sequenceid=415, filesize=12.0 K 2024-11-13T10:39:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,561 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 667b2b6bc191e21181054b3ac4dbe50b in 251ms, sequenceid=415, compaction requested=true 2024-11-13T10:39:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-13T10:39:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-13T10:39:36,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-13T10:39:36,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0400 sec 2024-11-13T10:39:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:36,567 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:36,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T10:39:36,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0520 sec 2024-11-13T10:39:36,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/3a87c38da80941c49b5a017f619035a5 is 50, key is test_row_0/A:col10/1731494376566/Put/seqid=0 2024-11-13T10:39:36,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742161_1337 (size=14741) 2024-11-13T10:39:36,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/3a87c38da80941c49b5a017f619035a5 2024-11-13T10:39:36,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494436596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494436600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/6d7238608412454fa13d755211a2b0c0 is 50, key is test_row_0/B:col10/1731494376566/Put/seqid=0 2024-11-13T10:39:36,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742162_1338 (size=12301) 2024-11-13T10:39:36,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/6d7238608412454fa13d755211a2b0c0 2024-11-13T10:39:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T10:39:36,673 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:36,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494436669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494436669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-13T10:39:36,686 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:36,686 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:36,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T10:39:36,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/a33c135fa2474ac4922069a7c8f6959f is 50, key is test_row_0/C:col10/1731494376566/Put/seqid=0 2024-11-13T10:39:36,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494436707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494436707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742163_1339 (size=12301) 2024-11-13T10:39:36,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/a33c135fa2474ac4922069a7c8f6959f 2024-11-13T10:39:36,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/3a87c38da80941c49b5a017f619035a5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/3a87c38da80941c49b5a017f619035a5 2024-11-13T10:39:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T10:39:36,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/3a87c38da80941c49b5a017f619035a5, entries=200, sequenceid=440, filesize=14.4 K 2024-11-13T10:39:36,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/6d7238608412454fa13d755211a2b0c0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6d7238608412454fa13d755211a2b0c0 2024-11-13T10:39:36,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T10:39:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:36,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:36,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6d7238608412454fa13d755211a2b0c0, entries=150, sequenceid=440, filesize=12.0 K 2024-11-13T10:39:36,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/a33c135fa2474ac4922069a7c8f6959f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/a33c135fa2474ac4922069a7c8f6959f 2024-11-13T10:39:36,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494436881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:36,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494436881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:36,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/a33c135fa2474ac4922069a7c8f6959f, entries=150, sequenceid=440, filesize=12.0 K 2024-11-13T10:39:36,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 667b2b6bc191e21181054b3ac4dbe50b in 340ms, sequenceid=440, compaction requested=true 2024-11-13T10:39:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:36,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:36,908 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:36,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:36,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:36,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:36,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:39:36,908 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:36,916 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:36,917 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/B is initiating minor compaction (all files) 2024-11-13T10:39:36,917 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/B in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:36,917 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a2fef13d929d4fef882cd58b71434e4e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6c8a3ef1cb464c7aaf87d4696dd0f53e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3e511ad753a5466abb1ab7b624ee20d6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6d7238608412454fa13d755211a2b0c0] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=48.9 K 2024-11-13T10:39:36,917 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:36,917 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/A is initiating minor compaction (all files) 2024-11-13T10:39:36,917 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/A in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:36,917 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6318d59e5c4741c1a31a9695751cbbe6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6c2f460de17b42dfa7cf732356f831f8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a1439489278d4b0a8bd82aa2eea13537, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/3a87c38da80941c49b5a017f619035a5] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=53.7 K 2024-11-13T10:39:36,918 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6318d59e5c4741c1a31a9695751cbbe6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731494373970 2024-11-13T10:39:36,918 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2fef13d929d4fef882cd58b71434e4e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731494373970 2024-11-13T10:39:36,918 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c2f460de17b42dfa7cf732356f831f8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731494375179 2024-11-13T10:39:36,919 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c8a3ef1cb464c7aaf87d4696dd0f53e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731494375179 2024-11-13T10:39:36,919 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting a1439489278d4b0a8bd82aa2eea13537, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731494375829 2024-11-13T10:39:36,919 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e511ad753a5466abb1ab7b624ee20d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731494375829 2024-11-13T10:39:36,925 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a87c38da80941c49b5a017f619035a5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731494376547 2024-11-13T10:39:36,925 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d7238608412454fa13d755211a2b0c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731494376547 2024-11-13T10:39:36,926 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:36,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T10:39:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:36,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a4c6db4538074e2a8e66087913a42f4e is 50, key is test_row_0/A:col10/1731494376924/Put/seqid=0 2024-11-13T10:39:36,949 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#A#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:36,950 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/186aef0e58d54fca8d43048079a8d15c is 50, key is test_row_0/A:col10/1731494376566/Put/seqid=0 2024-11-13T10:39:36,959 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#B#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:36,959 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/7f1a70145438425fbf6c0bc08a07e8d5 is 50, key is test_row_0/B:col10/1731494376566/Put/seqid=0 2024-11-13T10:39:36,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T10:39:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:36,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742165_1341 (size=13323) 2024-11-13T10:39:37,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742164_1340 (size=14741) 2024-11-13T10:39:37,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a4c6db4538074e2a8e66087913a42f4e 2024-11-13T10:39:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T10:39:37,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742166_1342 (size=13323) 2024-11-13T10:39:37,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/1c25954503a04ddf9f04b684da60af01 is 50, key is test_row_0/B:col10/1731494376924/Put/seqid=0 2024-11-13T10:39:37,033 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/7f1a70145438425fbf6c0bc08a07e8d5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7f1a70145438425fbf6c0bc08a07e8d5 2024-11-13T10:39:37,039 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/B of 667b2b6bc191e21181054b3ac4dbe50b into 7f1a70145438425fbf6c0bc08a07e8d5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:37,039 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:37,039 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/B, priority=12, startTime=1731494376908; duration=0sec 2024-11-13T10:39:37,039 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:37,039 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:B 2024-11-13T10:39:37,039 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:37,040 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:37,040 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 667b2b6bc191e21181054b3ac4dbe50b/C is initiating minor compaction (all files) 2024-11-13T10:39:37,040 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 667b2b6bc191e21181054b3ac4dbe50b/C in TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,040 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/68e44450ba594a0ea5ded9818ed05ba7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e42dc0a1ca6e42bf9129e8e26ccd09e5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/2e50674f7b3c44ab9338060408b3ba87, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/a33c135fa2474ac4922069a7c8f6959f] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp, totalSize=48.9 K 2024-11-13T10:39:37,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742167_1343 (size=12301) 2024-11-13T10:39:37,046 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68e44450ba594a0ea5ded9818ed05ba7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731494373970 2024-11-13T10:39:37,048 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e42dc0a1ca6e42bf9129e8e26ccd09e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731494375179 2024-11-13T10:39:37,049 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e50674f7b3c44ab9338060408b3ba87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731494375829 2024-11-13T10:39:37,050 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a33c135fa2474ac4922069a7c8f6959f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731494376547 2024-11-13T10:39:37,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494437054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494437052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,066 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 667b2b6bc191e21181054b3ac4dbe50b#C#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:37,067 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/f658d8d0bf394f86b8f4c153d85cb4ed is 50, key is test_row_0/C:col10/1731494376566/Put/seqid=0 2024-11-13T10:39:37,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742168_1344 (size=13323) 2024-11-13T10:39:37,104 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/f658d8d0bf394f86b8f4c153d85cb4ed as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f658d8d0bf394f86b8f4c153d85cb4ed 2024-11-13T10:39:37,110 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/C of 667b2b6bc191e21181054b3ac4dbe50b into f658d8d0bf394f86b8f4c153d85cb4ed(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:37,110 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:37,110 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/C, priority=12, startTime=1731494376908; duration=0sec 2024-11-13T10:39:37,110 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:37,110 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:C 2024-11-13T10:39:37,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T10:39:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494437166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494437166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494437197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494437200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T10:39:37,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:37,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T10:39:37,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494437381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494437381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/1c25954503a04ddf9f04b684da60af01 2024-11-13T10:39:37,453 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/186aef0e58d54fca8d43048079a8d15c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/186aef0e58d54fca8d43048079a8d15c 2024-11-13T10:39:37,470 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 667b2b6bc191e21181054b3ac4dbe50b/A of 667b2b6bc191e21181054b3ac4dbe50b into 186aef0e58d54fca8d43048079a8d15c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:37,470 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:37,470 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., storeName=667b2b6bc191e21181054b3ac4dbe50b/A, priority=12, startTime=1731494376907; duration=0sec 2024-11-13T10:39:37,470 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:37,470 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:A 2024-11-13T10:39:37,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/4b0f0a8eb59849c6a9f7ff7f2782626c is 50, key is test_row_0/C:col10/1731494376924/Put/seqid=0 2024-11-13T10:39:37,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T10:39:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742169_1345 (size=12301) 2024-11-13T10:39:37,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T10:39:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38790 deadline: 1731494437687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38784 deadline: 1731494437701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38814 deadline: 1731494437706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731494437708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:39:37,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T10:39:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T10:39:37,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/4b0f0a8eb59849c6a9f7ff7f2782626c 2024-11-13T10:39:37,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/a4c6db4538074e2a8e66087913a42f4e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a4c6db4538074e2a8e66087913a42f4e 2024-11-13T10:39:37,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a4c6db4538074e2a8e66087913a42f4e, entries=200, sequenceid=453, filesize=14.4 K 2024-11-13T10:39:37,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/1c25954503a04ddf9f04b684da60af01 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/1c25954503a04ddf9f04b684da60af01 2024-11-13T10:39:37,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/1c25954503a04ddf9f04b684da60af01, entries=150, sequenceid=453, filesize=12.0 K 2024-11-13T10:39:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/4b0f0a8eb59849c6a9f7ff7f2782626c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4b0f0a8eb59849c6a9f7ff7f2782626c 2024-11-13T10:39:37,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4b0f0a8eb59849c6a9f7ff7f2782626c, entries=150, sequenceid=453, filesize=12.0 K 2024-11-13T10:39:37,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 667b2b6bc191e21181054b3ac4dbe50b in 1025ms, sequenceid=453, compaction requested=false 2024-11-13T10:39:37,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:37,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T10:39:37,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:37,979 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T10:39:38,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8ece00c927d44fa5b1387e8ecffadde3 is 50, key is test_row_0/A:col10/1731494377041/Put/seqid=0 2024-11-13T10:39:38,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742170_1346 (size=12301) 2024-11-13T10:39:38,023 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8ece00c927d44fa5b1387e8ecffadde3 2024-11-13T10:39:38,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/63c1cf0a6a47448bb1dff58c70ad1c2a is 50, key is test_row_0/B:col10/1731494377041/Put/seqid=0 2024-11-13T10:39:38,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742171_1347 (size=12301) 2024-11-13T10:39:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:38,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. as already flushing 2024-11-13T10:39:38,268 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-13T10:39:38,268 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:38,268 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,268 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,269 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:38,269 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-11-13T10:39:38,269 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:38,269 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,269 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,269 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:38,272 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-11-13T10:39:38,272 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:38,272 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,272 INFO [Thread-1153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1153. 2024-11-13T10:39:38,272 DEBUG [Thread-1153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:38,272 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,272 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:38,273 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:38,273 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-13T10:39:38,273 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:38,273 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,274 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:38,275 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-13T10:39:38,275 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:38,275 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,275 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:38,279 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-13T10:39:38,279 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:38,279 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,279 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,279 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:38,452 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/63c1cf0a6a47448bb1dff58c70ad1c2a 2024-11-13T10:39:38,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/8be9d7792e694cc7953ece955c068432 is 50, key is test_row_0/C:col10/1731494377041/Put/seqid=0 2024-11-13T10:39:38,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742172_1348 (size=12301) 2024-11-13T10:39:38,718 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-11-13T10:39:38,718 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:38,718 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,718 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:38,728 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-13T10:39:38,728 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:38,728 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,728 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:38,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T10:39:38,864 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/8be9d7792e694cc7953ece955c068432 2024-11-13T10:39:38,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/8ece00c927d44fa5b1387e8ecffadde3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8ece00c927d44fa5b1387e8ecffadde3 2024-11-13T10:39:38,877 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8ece00c927d44fa5b1387e8ecffadde3, entries=150, sequenceid=479, filesize=12.0 K 2024-11-13T10:39:38,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/63c1cf0a6a47448bb1dff58c70ad1c2a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/63c1cf0a6a47448bb1dff58c70ad1c2a 2024-11-13T10:39:38,882 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/63c1cf0a6a47448bb1dff58c70ad1c2a, entries=150, sequenceid=479, filesize=12.0 K 2024-11-13T10:39:38,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/8be9d7792e694cc7953ece955c068432 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/8be9d7792e694cc7953ece955c068432 2024-11-13T10:39:38,886 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/8be9d7792e694cc7953ece955c068432, entries=150, sequenceid=479, filesize=12.0 K 2024-11-13T10:39:38,887 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=53.67 KB/54960 for 667b2b6bc191e21181054b3ac4dbe50b in 908ms, sequenceid=479, compaction requested=true 2024-11-13T10:39:38,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:38,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:38,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-13T10:39:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-13T10:39:38,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-13T10:39:38,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2020 sec 2024-11-13T10:39:38,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.2120 sec 2024-11-13T10:39:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T10:39:40,868 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:41,029 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T10:39:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:41,489 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:41,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 667b2b6bc191e21181054b3ac4dbe50b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T10:39:41,489 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-13T10:39:41,489 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:39:41,489 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:41,489 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:41,490 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3098 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3105 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3026 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3086 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3102 2024-11-13T10:39:41,490 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T10:39:41,490 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T10:39:41,490 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T10:39:41,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:41,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:41,491 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:39:41,491 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T10:39:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:41,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/49cd643e21394990beab14baa4d77a14 is 50, key is test_row_0/A:col10/1731494381487/Put/seqid=0 2024-11-13T10:39:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T10:39:41,508 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494381508"}]},"ts":"1731494381508"} 2024-11-13T10:39:41,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742173_1349 (size=12301) 2024-11-13T10:39:41,517 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T10:39:41,517 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T10:39:41,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T10:39:41,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=667b2b6bc191e21181054b3ac4dbe50b, UNASSIGN}] 2024-11-13T10:39:41,522 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=667b2b6bc191e21181054b3ac4dbe50b, UNASSIGN 2024-11-13T10:39:41,523 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=667b2b6bc191e21181054b3ac4dbe50b, regionState=CLOSING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:39:41,524 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44523 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=770665a7984d,39101,1731494292956, table=TestAcidGuarantees, region=667b2b6bc191e21181054b3ac4dbe50b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T10:39:41,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=667b2b6bc191e21181054b3ac4dbe50b, UNASSIGN because future has completed 2024-11-13T10:39:41,532 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T10:39:41,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:39:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T10:39:41,685 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:41,685 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T10:39:41,685 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 667b2b6bc191e21181054b3ac4dbe50b, disabling compactions & flushes 2024-11-13T10:39:41,686 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T10:39:41,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/49cd643e21394990beab14baa4d77a14 2024-11-13T10:39:41,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/2a635fbe861b4ddfabea22c2ae3178f1 is 50, key is test_row_0/B:col10/1731494381487/Put/seqid=0 2024-11-13T10:39:41,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742174_1350 (size=12301) 2024-11-13T10:39:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T10:39:42,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/2a635fbe861b4ddfabea22c2ae3178f1 2024-11-13T10:39:42,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/ce3012450e344aafae4a455312e985ea is 50, key is test_row_0/C:col10/1731494381487/Put/seqid=0 2024-11-13T10:39:42,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742175_1351 (size=12301) 2024-11-13T10:39:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T10:39:42,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/ce3012450e344aafae4a455312e985ea 2024-11-13T10:39:42,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/A/49cd643e21394990beab14baa4d77a14 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/49cd643e21394990beab14baa4d77a14 2024-11-13T10:39:42,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/49cd643e21394990beab14baa4d77a14, entries=150, sequenceid=491, filesize=12.0 K 2024-11-13T10:39:42,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/B/2a635fbe861b4ddfabea22c2ae3178f1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/2a635fbe861b4ddfabea22c2ae3178f1 2024-11-13T10:39:42,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/2a635fbe861b4ddfabea22c2ae3178f1, entries=150, sequenceid=491, filesize=12.0 K 2024-11-13T10:39:42,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/.tmp/C/ce3012450e344aafae4a455312e985ea as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/ce3012450e344aafae4a455312e985ea 2024-11-13T10:39:42,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/ce3012450e344aafae4a455312e985ea, entries=150, sequenceid=491, filesize=12.0 K 2024-11-13T10:39:42,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 667b2b6bc191e21181054b3ac4dbe50b in 1266ms, sequenceid=491, compaction requested=true 2024-11-13T10:39:42,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 667b2b6bc191e21181054b3ac4dbe50b: 2024-11-13T10:39:42,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:42,755 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:42,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:42,755 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:42,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:42,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:42,755 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. after waiting 0 ms 2024-11-13T10:39:42,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 667b2b6bc191e21181054b3ac4dbe50b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:42,755 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:42,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:42,755 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. because compaction request was cancelled 2024-11-13T10:39:42,755 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. because compaction request was cancelled 2024-11-13T10:39:42,755 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:A 2024-11-13T10:39:42,755 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:B 2024-11-13T10:39:42,755 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. because compaction request was cancelled 2024-11-13T10:39:42,755 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 667b2b6bc191e21181054b3ac4dbe50b:C 2024-11-13T10:39:42,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c1849d49c70746acae1bad566b9d4d3e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8e9f04a58a6841589e505db7869d8ed9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/4d3ba45b352d47be8273413f99537957, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ca876fd310f94d55a490d433fe6df7f3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/2463b907782f4a649745afb178d7b29b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8da1e45a16ff411ea9b23ba4c8aa7349, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d088021bb65649cbb84744f73878ffc6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d2ca92aa034248b79d7a31c645949ea5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d84330e01f5a4cdfaff29755b6e9aa27, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8b6b9657a68e43dbb4afdb3093c2ad91, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/5092882f5af64ac296ef6d1d6b8a461b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/55d1c31d17be414eb9c68dd3b5471f3e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/313c1d749b7645f8acb566ae61573914, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6e71bfc3c9314dfa922e4568f9aee4cf, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/b85c4dadb7b44203908388bffb9ed40e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ac0dd7eb656c4bce9e7f5eb8c7cb00fa, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/f7355c1c42c24f29bea05df09df8987b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/00c98d8516e344a88a4bb545979c6b4c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a5b2af8bdf5e4eed8ef14e58ad65fa98, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/35200cd549354a718847e447d8c3c6ae, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a87db24e67c542d182974c01cc71fa1a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/09c8a372e777405ba9381805f46987d2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d5eacd204d83424f83fd2a7910422469, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c52de2681cbf469388a9a0ad7fd406f3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/44184d0e382743b3a91fc777e51cc9a6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6318d59e5c4741c1a31a9695751cbbe6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/9ea5341b4d704fb498d29e579ee508f1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6c2f460de17b42dfa7cf732356f831f8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a1439489278d4b0a8bd82aa2eea13537, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/3a87c38da80941c49b5a017f619035a5] to archive 2024-11-13T10:39:42,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:39:42,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c1849d49c70746acae1bad566b9d4d3e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c1849d49c70746acae1bad566b9d4d3e 2024-11-13T10:39:42,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8e9f04a58a6841589e505db7869d8ed9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8e9f04a58a6841589e505db7869d8ed9 2024-11-13T10:39:42,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/4d3ba45b352d47be8273413f99537957 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/4d3ba45b352d47be8273413f99537957 2024-11-13T10:39:42,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ca876fd310f94d55a490d433fe6df7f3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ca876fd310f94d55a490d433fe6df7f3 2024-11-13T10:39:42,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/2463b907782f4a649745afb178d7b29b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/2463b907782f4a649745afb178d7b29b 2024-11-13T10:39:42,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8da1e45a16ff411ea9b23ba4c8aa7349 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8da1e45a16ff411ea9b23ba4c8aa7349 2024-11-13T10:39:42,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d088021bb65649cbb84744f73878ffc6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d088021bb65649cbb84744f73878ffc6 2024-11-13T10:39:42,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d2ca92aa034248b79d7a31c645949ea5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d2ca92aa034248b79d7a31c645949ea5 2024-11-13T10:39:42,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d84330e01f5a4cdfaff29755b6e9aa27 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d84330e01f5a4cdfaff29755b6e9aa27 2024-11-13T10:39:42,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8b6b9657a68e43dbb4afdb3093c2ad91 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8b6b9657a68e43dbb4afdb3093c2ad91 2024-11-13T10:39:42,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/5092882f5af64ac296ef6d1d6b8a461b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/5092882f5af64ac296ef6d1d6b8a461b 2024-11-13T10:39:42,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/55d1c31d17be414eb9c68dd3b5471f3e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/55d1c31d17be414eb9c68dd3b5471f3e 2024-11-13T10:39:42,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/313c1d749b7645f8acb566ae61573914 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/313c1d749b7645f8acb566ae61573914 2024-11-13T10:39:42,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6e71bfc3c9314dfa922e4568f9aee4cf to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6e71bfc3c9314dfa922e4568f9aee4cf 2024-11-13T10:39:42,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/b85c4dadb7b44203908388bffb9ed40e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/b85c4dadb7b44203908388bffb9ed40e 2024-11-13T10:39:42,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ac0dd7eb656c4bce9e7f5eb8c7cb00fa to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/ac0dd7eb656c4bce9e7f5eb8c7cb00fa 2024-11-13T10:39:42,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/f7355c1c42c24f29bea05df09df8987b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/f7355c1c42c24f29bea05df09df8987b 2024-11-13T10:39:42,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/00c98d8516e344a88a4bb545979c6b4c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/00c98d8516e344a88a4bb545979c6b4c 2024-11-13T10:39:42,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a5b2af8bdf5e4eed8ef14e58ad65fa98 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a5b2af8bdf5e4eed8ef14e58ad65fa98 2024-11-13T10:39:42,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/35200cd549354a718847e447d8c3c6ae to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/35200cd549354a718847e447d8c3c6ae 2024-11-13T10:39:42,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a87db24e67c542d182974c01cc71fa1a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a87db24e67c542d182974c01cc71fa1a 2024-11-13T10:39:42,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/09c8a372e777405ba9381805f46987d2 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/09c8a372e777405ba9381805f46987d2 2024-11-13T10:39:42,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d5eacd204d83424f83fd2a7910422469 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/d5eacd204d83424f83fd2a7910422469 2024-11-13T10:39:42,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c52de2681cbf469388a9a0ad7fd406f3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/c52de2681cbf469388a9a0ad7fd406f3 2024-11-13T10:39:42,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/44184d0e382743b3a91fc777e51cc9a6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/44184d0e382743b3a91fc777e51cc9a6 2024-11-13T10:39:42,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6318d59e5c4741c1a31a9695751cbbe6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6318d59e5c4741c1a31a9695751cbbe6 2024-11-13T10:39:42,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/9ea5341b4d704fb498d29e579ee508f1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/9ea5341b4d704fb498d29e579ee508f1 2024-11-13T10:39:42,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6c2f460de17b42dfa7cf732356f831f8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/6c2f460de17b42dfa7cf732356f831f8 2024-11-13T10:39:42,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a1439489278d4b0a8bd82aa2eea13537 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a1439489278d4b0a8bd82aa2eea13537 2024-11-13T10:39:42,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/3a87c38da80941c49b5a017f619035a5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/3a87c38da80941c49b5a017f619035a5 2024-11-13T10:39:42,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/0a4e78a7deed401d96b970d233b2697e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/5fe84387b8554e8bb33b75d72b313cf8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/daf928495ba548d1ba7aff9ebf939c18, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7601e1403bcd477088d813eac083a3d9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a87ef49f520147e492812960e990ce61, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7b7fca4342214538a5e6a676a5717a00, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7cc5c9ccad7b4c61a755f2213922636b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/c8b16543fc30466baa4635d66544782f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/d2c39340c1394f5a9effa91788f41845, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/114c49c049a34b5a8e37effd3185faf4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/65efe12677ac41b3b892f3ec9e935b97, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/69aab81bdb6749078ec44ddf124344a7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3c237cb0384c4c8f9048ba766889ff4f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e3fedd3da3fb4f2a9c3bbaf9ef1d17d8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/eff682f8777f413d971a43d793a57aec, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4f1a2231154f45a5bc42d8af69ba715a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/f3165ecb32d94cb682d4daed83c74e7f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e40903092bf94830b509979eafe1274a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/fce0cbd2c56144ad84c3bcc690081b5f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/497d0d7f1cec42d3abbc0ef886e04b79, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/bc0e20b5efd5496eb2c029fd2bc283bc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e7aa0e9533dd41599624ebdf7a63850a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/ec81616e5cc940be945d34934fbe0865, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/8008b1f64ed842ce99ceaad080107b5a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4677c60dc6c14217939ea7caf39fe54e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a2fef13d929d4fef882cd58b71434e4e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/1976f696f672479799c28819dc9d1c8d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6c8a3ef1cb464c7aaf87d4696dd0f53e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3e511ad753a5466abb1ab7b624ee20d6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6d7238608412454fa13d755211a2b0c0] to archive 2024-11-13T10:39:42,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:39:42,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/0a4e78a7deed401d96b970d233b2697e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/0a4e78a7deed401d96b970d233b2697e 2024-11-13T10:39:42,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/5fe84387b8554e8bb33b75d72b313cf8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/5fe84387b8554e8bb33b75d72b313cf8 2024-11-13T10:39:42,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/daf928495ba548d1ba7aff9ebf939c18 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/daf928495ba548d1ba7aff9ebf939c18 2024-11-13T10:39:42,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7601e1403bcd477088d813eac083a3d9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7601e1403bcd477088d813eac083a3d9 2024-11-13T10:39:42,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a87ef49f520147e492812960e990ce61 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a87ef49f520147e492812960e990ce61 2024-11-13T10:39:42,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7b7fca4342214538a5e6a676a5717a00 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7b7fca4342214538a5e6a676a5717a00 2024-11-13T10:39:42,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7cc5c9ccad7b4c61a755f2213922636b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7cc5c9ccad7b4c61a755f2213922636b 2024-11-13T10:39:42,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/c8b16543fc30466baa4635d66544782f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/c8b16543fc30466baa4635d66544782f 2024-11-13T10:39:42,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/d2c39340c1394f5a9effa91788f41845 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/d2c39340c1394f5a9effa91788f41845 2024-11-13T10:39:42,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/114c49c049a34b5a8e37effd3185faf4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/114c49c049a34b5a8e37effd3185faf4 2024-11-13T10:39:42,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/65efe12677ac41b3b892f3ec9e935b97 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/65efe12677ac41b3b892f3ec9e935b97 2024-11-13T10:39:42,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/69aab81bdb6749078ec44ddf124344a7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/69aab81bdb6749078ec44ddf124344a7 2024-11-13T10:39:42,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3c237cb0384c4c8f9048ba766889ff4f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3c237cb0384c4c8f9048ba766889ff4f 2024-11-13T10:39:42,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e3fedd3da3fb4f2a9c3bbaf9ef1d17d8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e3fedd3da3fb4f2a9c3bbaf9ef1d17d8 2024-11-13T10:39:42,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/eff682f8777f413d971a43d793a57aec to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/eff682f8777f413d971a43d793a57aec 2024-11-13T10:39:42,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4f1a2231154f45a5bc42d8af69ba715a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4f1a2231154f45a5bc42d8af69ba715a 2024-11-13T10:39:42,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/f3165ecb32d94cb682d4daed83c74e7f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/f3165ecb32d94cb682d4daed83c74e7f 2024-11-13T10:39:42,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e40903092bf94830b509979eafe1274a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e40903092bf94830b509979eafe1274a 2024-11-13T10:39:42,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/fce0cbd2c56144ad84c3bcc690081b5f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/fce0cbd2c56144ad84c3bcc690081b5f 2024-11-13T10:39:42,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/497d0d7f1cec42d3abbc0ef886e04b79 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/497d0d7f1cec42d3abbc0ef886e04b79 2024-11-13T10:39:42,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/bc0e20b5efd5496eb2c029fd2bc283bc to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/bc0e20b5efd5496eb2c029fd2bc283bc 2024-11-13T10:39:42,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e7aa0e9533dd41599624ebdf7a63850a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/e7aa0e9533dd41599624ebdf7a63850a 2024-11-13T10:39:42,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/ec81616e5cc940be945d34934fbe0865 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/ec81616e5cc940be945d34934fbe0865 2024-11-13T10:39:42,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/8008b1f64ed842ce99ceaad080107b5a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/8008b1f64ed842ce99ceaad080107b5a 2024-11-13T10:39:42,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4677c60dc6c14217939ea7caf39fe54e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/4677c60dc6c14217939ea7caf39fe54e 2024-11-13T10:39:42,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a2fef13d929d4fef882cd58b71434e4e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/a2fef13d929d4fef882cd58b71434e4e 2024-11-13T10:39:42,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/1976f696f672479799c28819dc9d1c8d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/1976f696f672479799c28819dc9d1c8d 2024-11-13T10:39:42,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6c8a3ef1cb464c7aaf87d4696dd0f53e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6c8a3ef1cb464c7aaf87d4696dd0f53e 2024-11-13T10:39:42,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3e511ad753a5466abb1ab7b624ee20d6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/3e511ad753a5466abb1ab7b624ee20d6 2024-11-13T10:39:42,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6d7238608412454fa13d755211a2b0c0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/6d7238608412454fa13d755211a2b0c0 2024-11-13T10:39:42,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/0df3be0a300d4d68b2dd4b2af7902de3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d665ee930cac4754923e818ff050cb26, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d477f65f377448198608504230b9d025, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f01e208eabff4398bd5cd6920565c2b4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e4f399ddde284432a28c2156dac60aa3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/659cc70e348c40fab13278c58039c3d2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b84185b8b96e463a8d4bf9fc55092b78, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fc6ebd64c5f24768bd6141e381e6cd69, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/04a0e162e38a4aad99df2a99eabbefef, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b0962f9aa8d14883873f73519d980c08, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/86f18f957514409d8433d962f0e3ba50, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b404f130875249f2aeb94a433d1a4641, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/576986e764094ed095fe9a9b21e50eda, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fd0d65eb325f44a4a16fb283fa55a4ae, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4126f287f4504b698262a47511fd479c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fa8be71bee2b4f6db3e5457c99db19bc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e61b6a14366d46319aaa43360db3efa7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/ccd3e6ed041b4118b1149b128bb8f7f6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f5099f425d004cc8b5204e05a3d8767e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/57a3802d9fe940ab96cccb369c2e5a08, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/9e73357a188f4181800f1864230b271c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/befdba0d34bf41f2b9462854b81f5e9a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4548a1e3413c462a9425419c6b99c520, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/1fbd8ca9e44a4edbaf1d800320e265d2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/95d1455cea3b49129dcf8054bab388ad, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/68e44450ba594a0ea5ded9818ed05ba7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/bbb390d84e7c4a9eb6a5b941133f1152, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e42dc0a1ca6e42bf9129e8e26ccd09e5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/2e50674f7b3c44ab9338060408b3ba87, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/a33c135fa2474ac4922069a7c8f6959f] to archive 2024-11-13T10:39:42,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:39:42,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/0df3be0a300d4d68b2dd4b2af7902de3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/0df3be0a300d4d68b2dd4b2af7902de3 2024-11-13T10:39:42,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d665ee930cac4754923e818ff050cb26 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d665ee930cac4754923e818ff050cb26 2024-11-13T10:39:42,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d477f65f377448198608504230b9d025 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/d477f65f377448198608504230b9d025 2024-11-13T10:39:42,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f01e208eabff4398bd5cd6920565c2b4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f01e208eabff4398bd5cd6920565c2b4 2024-11-13T10:39:42,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e4f399ddde284432a28c2156dac60aa3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e4f399ddde284432a28c2156dac60aa3 2024-11-13T10:39:42,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/659cc70e348c40fab13278c58039c3d2 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/659cc70e348c40fab13278c58039c3d2 2024-11-13T10:39:42,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b84185b8b96e463a8d4bf9fc55092b78 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b84185b8b96e463a8d4bf9fc55092b78 2024-11-13T10:39:42,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fc6ebd64c5f24768bd6141e381e6cd69 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fc6ebd64c5f24768bd6141e381e6cd69 2024-11-13T10:39:42,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/04a0e162e38a4aad99df2a99eabbefef to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/04a0e162e38a4aad99df2a99eabbefef 2024-11-13T10:39:42,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b0962f9aa8d14883873f73519d980c08 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b0962f9aa8d14883873f73519d980c08 2024-11-13T10:39:42,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/86f18f957514409d8433d962f0e3ba50 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/86f18f957514409d8433d962f0e3ba50 2024-11-13T10:39:42,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b404f130875249f2aeb94a433d1a4641 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/b404f130875249f2aeb94a433d1a4641 2024-11-13T10:39:42,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/576986e764094ed095fe9a9b21e50eda to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/576986e764094ed095fe9a9b21e50eda 2024-11-13T10:39:42,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fd0d65eb325f44a4a16fb283fa55a4ae to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fd0d65eb325f44a4a16fb283fa55a4ae 2024-11-13T10:39:42,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4126f287f4504b698262a47511fd479c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4126f287f4504b698262a47511fd479c 2024-11-13T10:39:42,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fa8be71bee2b4f6db3e5457c99db19bc to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/fa8be71bee2b4f6db3e5457c99db19bc 2024-11-13T10:39:42,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e61b6a14366d46319aaa43360db3efa7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e61b6a14366d46319aaa43360db3efa7 2024-11-13T10:39:42,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/ccd3e6ed041b4118b1149b128bb8f7f6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/ccd3e6ed041b4118b1149b128bb8f7f6 2024-11-13T10:39:42,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f5099f425d004cc8b5204e05a3d8767e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f5099f425d004cc8b5204e05a3d8767e 2024-11-13T10:39:42,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/57a3802d9fe940ab96cccb369c2e5a08 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/57a3802d9fe940ab96cccb369c2e5a08 2024-11-13T10:39:42,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/9e73357a188f4181800f1864230b271c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/9e73357a188f4181800f1864230b271c 2024-11-13T10:39:42,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/befdba0d34bf41f2b9462854b81f5e9a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/befdba0d34bf41f2b9462854b81f5e9a 2024-11-13T10:39:42,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4548a1e3413c462a9425419c6b99c520 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4548a1e3413c462a9425419c6b99c520 2024-11-13T10:39:42,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/1fbd8ca9e44a4edbaf1d800320e265d2 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/1fbd8ca9e44a4edbaf1d800320e265d2 2024-11-13T10:39:42,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/95d1455cea3b49129dcf8054bab388ad to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/95d1455cea3b49129dcf8054bab388ad 2024-11-13T10:39:42,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/68e44450ba594a0ea5ded9818ed05ba7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/68e44450ba594a0ea5ded9818ed05ba7 2024-11-13T10:39:42,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/bbb390d84e7c4a9eb6a5b941133f1152 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/bbb390d84e7c4a9eb6a5b941133f1152 2024-11-13T10:39:42,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e42dc0a1ca6e42bf9129e8e26ccd09e5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/e42dc0a1ca6e42bf9129e8e26ccd09e5 2024-11-13T10:39:42,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/2e50674f7b3c44ab9338060408b3ba87 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/2e50674f7b3c44ab9338060408b3ba87 2024-11-13T10:39:42,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/a33c135fa2474ac4922069a7c8f6959f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/a33c135fa2474ac4922069a7c8f6959f 2024-11-13T10:39:42,889 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/recovered.edits/494.seqid, newMaxSeqId=494, maxSeqId=1 2024-11-13T10:39:42,891 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b. 2024-11-13T10:39:42,891 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 667b2b6bc191e21181054b3ac4dbe50b: Waiting for close lock at 1731494381685Running coprocessor pre-close hooks at 1731494381685Disabling compacts and flushes for region at 1731494381685Disabling writes for close at 1731494382755 (+1070 ms)Writing region close event to WAL at 1731494382876 (+121 ms)Running coprocessor post-close hooks at 1731494382891 (+15 ms)Closed at 1731494382891 2024-11-13T10:39:42,893 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:42,895 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=667b2b6bc191e21181054b3ac4dbe50b, regionState=CLOSED 2024-11-13T10:39:42,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:39:42,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-13T10:39:42,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 667b2b6bc191e21181054b3ac4dbe50b, server=770665a7984d,39101,1731494292956 in 1.3650 sec 2024-11-13T10:39:42,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-13T10:39:42,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=667b2b6bc191e21181054b3ac4dbe50b, UNASSIGN in 1.3770 sec 2024-11-13T10:39:42,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-13T10:39:42,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3830 sec 2024-11-13T10:39:42,904 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494382903"}]},"ts":"1731494382903"} 2024-11-13T10:39:42,905 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T10:39:42,905 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T10:39:42,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4140 sec 2024-11-13T10:39:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T10:39:43,648 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:43,648 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T10:39:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:43,650 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T10:39:43,650 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:43,652 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:43,654 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/recovered.edits] 2024-11-13T10:39:43,657 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/186aef0e58d54fca8d43048079a8d15c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/186aef0e58d54fca8d43048079a8d15c 2024-11-13T10:39:43,659 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/49cd643e21394990beab14baa4d77a14 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/49cd643e21394990beab14baa4d77a14 2024-11-13T10:39:43,660 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8ece00c927d44fa5b1387e8ecffadde3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/8ece00c927d44fa5b1387e8ecffadde3 2024-11-13T10:39:43,661 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a4c6db4538074e2a8e66087913a42f4e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/A/a4c6db4538074e2a8e66087913a42f4e 2024-11-13T10:39:43,663 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/1c25954503a04ddf9f04b684da60af01 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/1c25954503a04ddf9f04b684da60af01 2024-11-13T10:39:43,664 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/2a635fbe861b4ddfabea22c2ae3178f1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/2a635fbe861b4ddfabea22c2ae3178f1 2024-11-13T10:39:43,666 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/63c1cf0a6a47448bb1dff58c70ad1c2a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/63c1cf0a6a47448bb1dff58c70ad1c2a 2024-11-13T10:39:43,668 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7f1a70145438425fbf6c0bc08a07e8d5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/B/7f1a70145438425fbf6c0bc08a07e8d5 2024-11-13T10:39:43,671 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4b0f0a8eb59849c6a9f7ff7f2782626c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/4b0f0a8eb59849c6a9f7ff7f2782626c 2024-11-13T10:39:43,672 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/8be9d7792e694cc7953ece955c068432 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/8be9d7792e694cc7953ece955c068432 2024-11-13T10:39:43,673 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/ce3012450e344aafae4a455312e985ea to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/ce3012450e344aafae4a455312e985ea 2024-11-13T10:39:43,674 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f658d8d0bf394f86b8f4c153d85cb4ed to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/C/f658d8d0bf394f86b8f4c153d85cb4ed 2024-11-13T10:39:43,679 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/recovered.edits/494.seqid to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b/recovered.edits/494.seqid 2024-11-13T10:39:43,680 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/667b2b6bc191e21181054b3ac4dbe50b 2024-11-13T10:39:43,680 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T10:39:43,682 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:43,689 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T10:39:43,692 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T10:39:43,693 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:43,693 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T10:39:43,693 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731494383693"}]},"ts":"9223372036854775807"} 2024-11-13T10:39:43,696 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T10:39:43,696 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 667b2b6bc191e21181054b3ac4dbe50b, NAME => 'TestAcidGuarantees,,1731494357066.667b2b6bc191e21181054b3ac4dbe50b.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T10:39:43,696 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T10:39:43,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731494383696"}]},"ts":"9223372036854775807"} 2024-11-13T10:39:43,698 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T10:39:43,699 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:43,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-13T10:39:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T10:39:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T10:39:43,761 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:43,769 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 238) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2019480356_22 at /127.0.0.1:52766 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=825 (was 851), ProcessCount=11 (was 12), AvailableMemoryMB=906 (was 453) - AvailableMemoryMB LEAK? - 2024-11-13T10:39:43,784 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=825, ProcessCount=11, AvailableMemoryMB=905 2024-11-13T10:39:43,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T10:39:43,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T10:39:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:43,787 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T10:39:43,787 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:43,787 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-13T10:39:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T10:39:43,788 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T10:39:43,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742176_1352 (size=959) 2024-11-13T10:39:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T10:39:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T10:39:44,200 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617 2024-11-13T10:39:44,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742177_1353 (size=53) 2024-11-13T10:39:44,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:39:44,208 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0b3f34f131c1584553eed8127d585c45, disabling compactions & flushes 2024-11-13T10:39:44,208 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:44,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:44,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. after waiting 0 ms 2024-11-13T10:39:44,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:44,209 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:44,209 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0b3f34f131c1584553eed8127d585c45: Waiting for close lock at 1731494384208Disabling compacts and flushes for region at 1731494384208Disabling writes for close at 1731494384209 (+1 ms)Writing region close event to WAL at 1731494384209Closed at 1731494384209 2024-11-13T10:39:44,210 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T10:39:44,210 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731494384210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731494384210"}]},"ts":"1731494384210"} 2024-11-13T10:39:44,212 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T10:39:44,212 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T10:39:44,213 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494384212"}]},"ts":"1731494384212"} 2024-11-13T10:39:44,214 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T10:39:44,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, ASSIGN}] 2024-11-13T10:39:44,215 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, ASSIGN 2024-11-13T10:39:44,216 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, ASSIGN; state=OFFLINE, location=770665a7984d,39101,1731494292956; forceNewPlan=false, retain=false 2024-11-13T10:39:44,367 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=0b3f34f131c1584553eed8127d585c45, regionState=OPENING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:39:44,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, ASSIGN because future has completed 2024-11-13T10:39:44,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:39:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T10:39:44,523 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:44,523 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:39:44,523 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,523 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:39:44,523 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,523 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,524 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,525 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b3f34f131c1584553eed8127d585c45 columnFamilyName A 2024-11-13T10:39:44,526 DEBUG [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:44,526 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(327): Store=0b3f34f131c1584553eed8127d585c45/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:39:44,526 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,527 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b3f34f131c1584553eed8127d585c45 columnFamilyName B 2024-11-13T10:39:44,527 DEBUG [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:44,527 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(327): Store=0b3f34f131c1584553eed8127d585c45/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:39:44,527 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,528 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b3f34f131c1584553eed8127d585c45 columnFamilyName C 2024-11-13T10:39:44,528 DEBUG [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:44,529 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(327): Store=0b3f34f131c1584553eed8127d585c45/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:39:44,529 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,529 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,529 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,530 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,530 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,531 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:39:44,532 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,533 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T10:39:44,534 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 0b3f34f131c1584553eed8127d585c45; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62391871, jitterRate=-0.07028867304325104}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:39:44,534 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:44,534 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 0b3f34f131c1584553eed8127d585c45: Running coprocessor pre-open hook at 1731494384523Writing region info on filesystem at 1731494384523Initializing all the Stores at 1731494384524 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494384524Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494384524Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494384524Cleaning up temporary data from old regions at 1731494384530 (+6 ms)Running coprocessor post-open hooks at 1731494384534 (+4 ms)Region opened successfully at 1731494384534 2024-11-13T10:39:44,535 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., pid=89, masterSystemTime=1731494384520 2024-11-13T10:39:44,537 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:44,537 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:44,537 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=0b3f34f131c1584553eed8127d585c45, regionState=OPEN, openSeqNum=2, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:39:44,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:39:44,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-13T10:39:44,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 in 170 msec 2024-11-13T10:39:44,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-13T10:39:44,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, ASSIGN in 327 msec 2024-11-13T10:39:44,543 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T10:39:44,543 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494384543"}]},"ts":"1731494384543"} 2024-11-13T10:39:44,544 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T10:39:44,545 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T10:39:44,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 760 msec 2024-11-13T10:39:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T10:39:44,928 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:44,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecc882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:44,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:44,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:44,931 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff16d77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:44,933 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e2ba3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:44,934 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:44,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:44,937 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T10:39:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 770665a7984d,44523,1731494292131 2024-11-13T10:39:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@69dae0e8 2024-11-13T10:39:44,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T10:39:44,938 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T10:39:44,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T10:39:44,940 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T10:39:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T10:39:44,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742178_1354 (size=995) 2024-11-13T10:39:44,954 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T10:39:44,954 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T10:39:44,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T10:39:44,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, REOPEN/MOVE}] 2024-11-13T10:39:44,960 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, REOPEN/MOVE 2024-11-13T10:39:44,961 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=0b3f34f131c1584553eed8127d585c45, regionState=CLOSING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:39:44,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, REOPEN/MOVE because future has completed 2024-11-13T10:39:44,964 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T10:39:44,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:39:45,117 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,117 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T10:39:45,117 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 0b3f34f131c1584553eed8127d585c45, disabling compactions & flushes 2024-11-13T10:39:45,117 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,117 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,117 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. after waiting 0 ms 2024-11-13T10:39:45,117 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,133 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T10:39:45,134 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,134 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 0b3f34f131c1584553eed8127d585c45: Waiting for close lock at 1731494385117Running coprocessor pre-close hooks at 1731494385117Disabling compacts and flushes for region at 1731494385117Disabling writes for close at 1731494385117Writing region close event to WAL at 1731494385117Running coprocessor post-close hooks at 1731494385134 (+17 ms)Closed at 1731494385134 2024-11-13T10:39:45,134 WARN [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 0b3f34f131c1584553eed8127d585c45 to self. 2024-11-13T10:39:45,136 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,141 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=0b3f34f131c1584553eed8127d585c45, regionState=CLOSED 2024-11-13T10:39:45,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:39:45,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-13T10:39:45,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 in 179 msec 2024-11-13T10:39:45,146 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, REOPEN/MOVE; state=CLOSED, location=770665a7984d,39101,1731494292956; forceNewPlan=false, retain=true 2024-11-13T10:39:45,296 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=0b3f34f131c1584553eed8127d585c45, regionState=OPENING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, REOPEN/MOVE because future has completed 2024-11-13T10:39:45,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:39:45,452 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,453 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:39:45,453 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,453 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:39:45,453 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,453 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,455 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,455 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b3f34f131c1584553eed8127d585c45 columnFamilyName A 2024-11-13T10:39:45,456 DEBUG [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:45,457 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(327): Store=0b3f34f131c1584553eed8127d585c45/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:39:45,457 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,458 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b3f34f131c1584553eed8127d585c45 columnFamilyName B 2024-11-13T10:39:45,458 DEBUG [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:45,458 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(327): Store=0b3f34f131c1584553eed8127d585c45/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:39:45,458 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,459 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b3f34f131c1584553eed8127d585c45 columnFamilyName C 2024-11-13T10:39:45,459 DEBUG [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:45,459 INFO [StoreOpener-0b3f34f131c1584553eed8127d585c45-1 {}] regionserver.HStore(327): Store=0b3f34f131c1584553eed8127d585c45/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:39:45,459 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,460 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,460 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,461 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,461 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,461 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:39:45,462 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,463 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 0b3f34f131c1584553eed8127d585c45; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64695099, jitterRate=-0.03596790134906769}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:39:45,463 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,464 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 0b3f34f131c1584553eed8127d585c45: Running coprocessor pre-open hook at 1731494385453Writing region info on filesystem at 1731494385453Initializing all the Stores at 1731494385454 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494385454Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494385454Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494385454Cleaning up temporary data from old regions at 1731494385461 (+7 ms)Running coprocessor post-open hooks at 1731494385463 (+2 ms)Region opened successfully at 1731494385463 2024-11-13T10:39:45,464 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., pid=94, masterSystemTime=1731494385450 2024-11-13T10:39:45,466 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,466 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,466 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=0b3f34f131c1584553eed8127d585c45, regionState=OPEN, openSeqNum=5, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:39:45,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-13T10:39:45,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 in 170 msec 2024-11-13T10:39:45,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-13T10:39:45,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, REOPEN/MOVE in 510 msec 2024-11-13T10:39:45,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-13T10:39:45,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-13T10:39:45,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 534 msec 2024-11-13T10:39:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-13T10:39:45,476 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:45,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f86bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,476 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:45,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:45,477 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:45,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:45,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:45,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6793e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:45,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:45,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:45,479 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@497c1cea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:45,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:45,481 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a86e44c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:45,482 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf3055a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:45,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:45,484 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c88cea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:45,485 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d2f9c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:45,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:45,486 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb24f9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:45,488 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749e45a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:45,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:45,493 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39bb6bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:45,495 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb2cbaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:45,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:45,496 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4547ba02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:45,497 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db7aab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,504 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:45,504 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:45,505 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@557467c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:45,506 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:45,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680fa5f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,508 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:45,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:45,509 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:45,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:45,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:45,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d40e0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:45,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:45,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:45,510 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2586db01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:45,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:45,513 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:45,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74373e87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:45,515 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:45,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fe79e4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:39:45,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:39:45,517 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:39:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:39:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:39:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@141d5da6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:39:45,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:39:45,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:39:45,519 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:39:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c728d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:39:45,520 DEBUG [Thread-1613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:45,520 DEBUG [Thread-1621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:45,521 DEBUG [Thread-1617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:45,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:45,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:45,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:45,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:45,523 DEBUG [Thread-1608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:45,524 DEBUG [Thread-1615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:45,524 DEBUG [Thread-1606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:45,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:39:45,525 DEBUG [Thread-1604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:45,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:45,526 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:39:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:45,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:39:45,527 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-13T10:39:45,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:45,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:45,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:45,528 DEBUG [Thread-1602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T10:39:45,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:45,529 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:45,529 DEBUG [Thread-1619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:39:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:45,530 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:45,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:39:45,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:39:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:45,531 DEBUG [Thread-1602 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:39:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:45,531 DEBUG [Thread-1602 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:39:45,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:39:45,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:45,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:39:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:39:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:39:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:39:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:39:45,544 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:45,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:39:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:45,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111312fab2aaf9f14041b139f9bb7db2623f_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494385542/Put/seqid=0 2024-11-13T10:39:45,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742179_1355 (size=14594) 2024-11-13T10:39:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T10:39:45,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494445649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494445653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494445657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494445658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494445658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T10:39:45,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:45,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:45,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:45,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494445777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494445777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494445777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494445777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494445777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T10:39:45,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T10:39:45,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:45,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:45,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:45,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:45,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494445988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494445988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494445988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:46,002 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:46,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T10:39:46,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:46,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111312fab2aaf9f14041b139f9bb7db2623f_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111312fab2aaf9f14041b139f9bb7db2623f_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:46,010 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/61b5033932e446429a89e61a7a6cb423, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:46,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/61b5033932e446429a89e61a7a6cb423 is 175, key is test_row_0/A:col10/1731494385542/Put/seqid=0 2024-11-13T10:39:46,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494446000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494446000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742180_1356 (size=39549) 2024-11-13T10:39:46,076 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/61b5033932e446429a89e61a7a6cb423 2024-11-13T10:39:46,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/7a14686e2d7b41e79506b9829ed73a11 is 50, key is test_row_0/B:col10/1731494385542/Put/seqid=0 2024-11-13T10:39:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T10:39:46,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T10:39:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742181_1357 (size=12001) 2024-11-13T10:39:46,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494446307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494446307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T10:39:46,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:46,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494446312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:46,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494446321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494446327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T10:39:46,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:46,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/7a14686e2d7b41e79506b9829ed73a11 2024-11-13T10:39:46,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T10:39:46,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:46,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:46,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ee48d2b779344cedb628833bed1147a5 is 50, key is test_row_0/C:col10/1731494385542/Put/seqid=0 2024-11-13T10:39:46,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742182_1358 (size=12001) 2024-11-13T10:39:46,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ee48d2b779344cedb628833bed1147a5 2024-11-13T10:39:46,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/61b5033932e446429a89e61a7a6cb423 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/61b5033932e446429a89e61a7a6cb423 2024-11-13T10:39:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T10:39:46,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/61b5033932e446429a89e61a7a6cb423, entries=200, sequenceid=15, filesize=38.6 K 2024-11-13T10:39:46,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/7a14686e2d7b41e79506b9829ed73a11 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/7a14686e2d7b41e79506b9829ed73a11 2024-11-13T10:39:46,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/7a14686e2d7b41e79506b9829ed73a11, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T10:39:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ee48d2b779344cedb628833bed1147a5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ee48d2b779344cedb628833bed1147a5 2024-11-13T10:39:46,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ee48d2b779344cedb628833bed1147a5, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T10:39:46,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0b3f34f131c1584553eed8127d585c45 in 1159ms, sequenceid=15, compaction requested=false 2024-11-13T10:39:46,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:46,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T10:39:46,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,787 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:39:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dd85121a0a9b40568144f59dc5179ce4_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494385651/Put/seqid=0 2024-11-13T10:39:46,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742183_1359 (size=12154) 2024-11-13T10:39:46,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:46,802 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dd85121a0a9b40568144f59dc5179ce4_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dd85121a0a9b40568144f59dc5179ce4_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:46,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f6393b1e29cc444c99a7b57b10170210, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f6393b1e29cc444c99a7b57b10170210 is 175, key is test_row_0/A:col10/1731494385651/Put/seqid=0 2024-11-13T10:39:46,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742184_1360 (size=30955) 2024-11-13T10:39:46,810 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f6393b1e29cc444c99a7b57b10170210 2024-11-13T10:39:46,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/e622d20580884c599210e123e8002aec is 50, key is test_row_0/B:col10/1731494385651/Put/seqid=0 2024-11-13T10:39:46,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742185_1361 (size=12001) 2024-11-13T10:39:46,828 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/e622d20580884c599210e123e8002aec 2024-11-13T10:39:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:46,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:46,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/94874eeb7e18400a8fc039b27cd7607e is 50, key is test_row_0/C:col10/1731494385651/Put/seqid=0 2024-11-13T10:39:46,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742186_1362 (size=12001) 2024-11-13T10:39:46,841 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/94874eeb7e18400a8fc039b27cd7607e 2024-11-13T10:39:46,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f6393b1e29cc444c99a7b57b10170210 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f6393b1e29cc444c99a7b57b10170210 2024-11-13T10:39:46,852 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f6393b1e29cc444c99a7b57b10170210, entries=150, sequenceid=40, filesize=30.2 K 2024-11-13T10:39:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/e622d20580884c599210e123e8002aec as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e622d20580884c599210e123e8002aec 2024-11-13T10:39:46,858 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e622d20580884c599210e123e8002aec, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T10:39:46,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/94874eeb7e18400a8fc039b27cd7607e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/94874eeb7e18400a8fc039b27cd7607e 2024-11-13T10:39:46,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494446846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:46,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:46,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494446848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:46,866 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/94874eeb7e18400a8fc039b27cd7607e, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T10:39:46,867 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0b3f34f131c1584553eed8127d585c45 in 80ms, sequenceid=40, compaction requested=false 2024-11-13T10:39:46,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:46,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-13T10:39:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-13T10:39:46,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-13T10:39:46,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3380 sec 2024-11-13T10:39:46,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.3470 sec 2024-11-13T10:39:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:46,877 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:46,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T10:39:46,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131a6ef7fa4ef247bd9535269125c0846d_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494386869/Put/seqid=0 2024-11-13T10:39:46,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742187_1363 (size=12154) 2024-11-13T10:39:46,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:46,908 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131a6ef7fa4ef247bd9535269125c0846d_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131a6ef7fa4ef247bd9535269125c0846d_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:46,909 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/e5bdf5de0c8e407c89b7c198a1cda307, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:46,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/e5bdf5de0c8e407c89b7c198a1cda307 is 175, key is test_row_0/A:col10/1731494386869/Put/seqid=0 2024-11-13T10:39:46,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742188_1364 (size=30955) 2024-11-13T10:39:46,916 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/e5bdf5de0c8e407c89b7c198a1cda307 2024-11-13T10:39:46,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/391ba4692c63461b819b10cb0061e1f8 is 50, key is test_row_0/B:col10/1731494386869/Put/seqid=0 2024-11-13T10:39:46,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742189_1365 (size=12001) 2024-11-13T10:39:46,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/391ba4692c63461b819b10cb0061e1f8 2024-11-13T10:39:46,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ff7dd41b89604d3dbf5359c7f0e1e33a is 50, key is test_row_0/C:col10/1731494386869/Put/seqid=0 2024-11-13T10:39:46,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742190_1366 (size=12001) 2024-11-13T10:39:46,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ff7dd41b89604d3dbf5359c7f0e1e33a 2024-11-13T10:39:46,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/e5bdf5de0c8e407c89b7c198a1cda307 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/e5bdf5de0c8e407c89b7c198a1cda307 2024-11-13T10:39:46,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/e5bdf5de0c8e407c89b7c198a1cda307, entries=150, sequenceid=54, filesize=30.2 K 2024-11-13T10:39:46,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/391ba4692c63461b819b10cb0061e1f8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/391ba4692c63461b819b10cb0061e1f8 2024-11-13T10:39:46,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/391ba4692c63461b819b10cb0061e1f8, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T10:39:46,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ff7dd41b89604d3dbf5359c7f0e1e33a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ff7dd41b89604d3dbf5359c7f0e1e33a 2024-11-13T10:39:46,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ff7dd41b89604d3dbf5359c7f0e1e33a, entries=150, sequenceid=54, filesize=11.7 K 2024-11-13T10:39:46,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 0b3f34f131c1584553eed8127d585c45 in 88ms, sequenceid=54, compaction requested=true 2024-11-13T10:39:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:46,966 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:46,966 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:46,967 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:46,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T10:39:46,967 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:46,967 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/A is initiating minor compaction (all files) 2024-11-13T10:39:46,967 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/A in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,967 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/61b5033932e446429a89e61a7a6cb423, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f6393b1e29cc444c99a7b57b10170210, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/e5bdf5de0c8e407c89b7c198a1cda307] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=99.1 K 2024-11-13T10:39:46,967 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,967 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/61b5033932e446429a89e61a7a6cb423, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f6393b1e29cc444c99a7b57b10170210, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/e5bdf5de0c8e407c89b7c198a1cda307] 2024-11-13T10:39:46,967 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61b5033932e446429a89e61a7a6cb423, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731494385532 2024-11-13T10:39:46,968 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:46,968 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/B is initiating minor compaction (all files) 2024-11-13T10:39:46,968 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/B in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:46,968 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/7a14686e2d7b41e79506b9829ed73a11, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e622d20580884c599210e123e8002aec, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/391ba4692c63461b819b10cb0061e1f8] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=35.2 K 2024-11-13T10:39:46,968 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6393b1e29cc444c99a7b57b10170210, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731494385647 2024-11-13T10:39:46,968 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5bdf5de0c8e407c89b7c198a1cda307, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494386857 2024-11-13T10:39:46,968 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a14686e2d7b41e79506b9829ed73a11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731494385534 2024-11-13T10:39:46,969 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e622d20580884c599210e123e8002aec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731494385647 2024-11-13T10:39:46,970 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 391ba4692c63461b819b10cb0061e1f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494386857 2024-11-13T10:39:46,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f29df27149b24666b71f3e81ac8bdf7d_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494386962/Put/seqid=0 2024-11-13T10:39:46,983 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:46,985 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#B#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:46,985 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113c482a467a7bc4ecfab378aa1079ac1a8_0b3f34f131c1584553eed8127d585c45 store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:46,986 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/bcdfbbff63ff42808daae1f3720308f6 is 50, key is test_row_0/B:col10/1731494386869/Put/seqid=0 2024-11-13T10:39:46,988 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113c482a467a7bc4ecfab378aa1079ac1a8_0b3f34f131c1584553eed8127d585c45, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:46,988 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c482a467a7bc4ecfab378aa1079ac1a8_0b3f34f131c1584553eed8127d585c45 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:47,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742193_1369 (size=4469) 2024-11-13T10:39:47,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742192_1368 (size=12104) 2024-11-13T10:39:47,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742191_1367 (size=14594) 2024-11-13T10:39:47,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494447003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494447003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494447003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494447004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494447006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494447127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494447127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494447127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494447127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494447127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494447347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494447347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494447347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494447347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494447347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,403 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#A#compaction#316 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:47,403 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/b1606424455b4346bffe09897a5a6a4a is 175, key is test_row_0/A:col10/1731494386869/Put/seqid=0 2024-11-13T10:39:47,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742194_1370 (size=31058) 2024-11-13T10:39:47,415 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/bcdfbbff63ff42808daae1f3720308f6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/bcdfbbff63ff42808daae1f3720308f6 2024-11-13T10:39:47,419 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/B of 0b3f34f131c1584553eed8127d585c45 into bcdfbbff63ff42808daae1f3720308f6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:47,419 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:47,419 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/B, priority=13, startTime=1731494386966; duration=0sec 2024-11-13T10:39:47,419 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:47,419 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:B 2024-11-13T10:39:47,419 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:47,419 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:47,420 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:47,420 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/C is initiating minor compaction (all files) 2024-11-13T10:39:47,420 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/C in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:47,420 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ee48d2b779344cedb628833bed1147a5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/94874eeb7e18400a8fc039b27cd7607e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ff7dd41b89604d3dbf5359c7f0e1e33a] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=35.2 K 2024-11-13T10:39:47,421 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ee48d2b779344cedb628833bed1147a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731494385534 2024-11-13T10:39:47,421 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 94874eeb7e18400a8fc039b27cd7607e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731494385647 2024-11-13T10:39:47,421 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ff7dd41b89604d3dbf5359c7f0e1e33a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494386857 2024-11-13T10:39:47,422 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f29df27149b24666b71f3e81ac8bdf7d_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f29df27149b24666b71f3e81ac8bdf7d_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:47,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/287977efe18f48c3a83f6949c3d0f107, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:47,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/287977efe18f48c3a83f6949c3d0f107 is 175, key is test_row_0/A:col10/1731494386962/Put/seqid=0 2024-11-13T10:39:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742195_1371 (size=39549) 2024-11-13T10:39:47,429 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:47,430 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/2b8fb9f17a524d8d90aa67db31ff2fd4 is 50, key is test_row_0/C:col10/1731494386869/Put/seqid=0 2024-11-13T10:39:47,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/287977efe18f48c3a83f6949c3d0f107 2024-11-13T10:39:47,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742196_1372 (size=12104) 2024-11-13T10:39:47,444 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/2b8fb9f17a524d8d90aa67db31ff2fd4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/2b8fb9f17a524d8d90aa67db31ff2fd4 2024-11-13T10:39:47,449 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/C of 0b3f34f131c1584553eed8127d585c45 into 2b8fb9f17a524d8d90aa67db31ff2fd4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:47,449 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:47,449 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/C, priority=13, startTime=1731494386966; duration=0sec 2024-11-13T10:39:47,449 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:47,449 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:C 2024-11-13T10:39:47,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/ef00af45f8fb4640b76d1b138e5b6f3a is 50, key is test_row_0/B:col10/1731494386962/Put/seqid=0 2024-11-13T10:39:47,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742197_1373 (size=12001) 2024-11-13T10:39:47,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494447666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494447666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T10:39:47,678 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:47,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-13T10:39:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T10:39:47,681 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:47,681 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:47,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494447677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494447678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494447678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:47,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T10:39:47,827 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/b1606424455b4346bffe09897a5a6a4a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b1606424455b4346bffe09897a5a6a4a 2024-11-13T10:39:47,833 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/A of 0b3f34f131c1584553eed8127d585c45 into b1606424455b4346bffe09897a5a6a4a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:47,833 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:47,833 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/A, priority=13, startTime=1731494386966; duration=0sec 2024-11-13T10:39:47,833 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:47,834 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:A 2024-11-13T10:39:47,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T10:39:47,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:47,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:47,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:47,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:47,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:47,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/ef00af45f8fb4640b76d1b138e5b6f3a 2024-11-13T10:39:47,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/7737e125c6634f1584f6697aee4033ea is 50, key is test_row_0/C:col10/1731494386962/Put/seqid=0 2024-11-13T10:39:47,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742198_1374 (size=12001) 2024-11-13T10:39:47,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T10:39:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:47,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T10:39:48,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T10:39:48,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:48,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:48,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:48,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:48,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:48,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494448177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:48,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494448177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:48,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494448187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:48,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494448196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:48,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494448196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:48,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T10:39:48,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:48,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:48,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:48,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:48,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T10:39:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:48,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/7737e125c6634f1584f6697aee4033ea 2024-11-13T10:39:48,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/287977efe18f48c3a83f6949c3d0f107 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/287977efe18f48c3a83f6949c3d0f107 2024-11-13T10:39:48,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/287977efe18f48c3a83f6949c3d0f107, entries=200, sequenceid=77, filesize=38.6 K 2024-11-13T10:39:48,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/ef00af45f8fb4640b76d1b138e5b6f3a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/ef00af45f8fb4640b76d1b138e5b6f3a 2024-11-13T10:39:48,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/ef00af45f8fb4640b76d1b138e5b6f3a, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T10:39:48,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/7737e125c6634f1584f6697aee4033ea as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/7737e125c6634f1584f6697aee4033ea 2024-11-13T10:39:48,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/7737e125c6634f1584f6697aee4033ea, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T10:39:48,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 0b3f34f131c1584553eed8127d585c45 in 1401ms, sequenceid=77, compaction requested=false 2024-11-13T10:39:48,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:48,426 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T10:39:48,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T10:39:48,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:48,459 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T10:39:48,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133792c0c70ac54065b56a5c7b22cdd8a9_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494386999/Put/seqid=0 2024-11-13T10:39:48,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742199_1375 (size=12154) 2024-11-13T10:39:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:48,482 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133792c0c70ac54065b56a5c7b22cdd8a9_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133792c0c70ac54065b56a5c7b22cdd8a9_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:48,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/20726465897f4894834cdf8868e17da4, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:48,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/20726465897f4894834cdf8868e17da4 is 175, key is test_row_0/A:col10/1731494386999/Put/seqid=0 2024-11-13T10:39:48,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742200_1376 (size=30955) 2024-11-13T10:39:48,494 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/20726465897f4894834cdf8868e17da4 2024-11-13T10:39:48,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/23cb9717b85b49b7bef321f117a90634 is 50, key is test_row_0/B:col10/1731494386999/Put/seqid=0 2024-11-13T10:39:48,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742201_1377 (size=12001) 2024-11-13T10:39:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T10:39:48,914 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/23cb9717b85b49b7bef321f117a90634 2024-11-13T10:39:48,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/5394d1bcd7a64430955ff4991d69e0c1 is 50, key is test_row_0/C:col10/1731494386999/Put/seqid=0 2024-11-13T10:39:48,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742202_1378 (size=12001) 2024-11-13T10:39:49,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:49,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494449290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494449291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494449291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494449313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494449314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,394 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/5394d1bcd7a64430955ff4991d69e0c1 2024-11-13T10:39:49,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/20726465897f4894834cdf8868e17da4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/20726465897f4894834cdf8868e17da4 2024-11-13T10:39:49,413 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/20726465897f4894834cdf8868e17da4, entries=150, sequenceid=93, filesize=30.2 K 2024-11-13T10:39:49,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/23cb9717b85b49b7bef321f117a90634 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/23cb9717b85b49b7bef321f117a90634 2024-11-13T10:39:49,425 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/23cb9717b85b49b7bef321f117a90634, entries=150, sequenceid=93, filesize=11.7 K 2024-11-13T10:39:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494449418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494449418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/5394d1bcd7a64430955ff4991d69e0c1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/5394d1bcd7a64430955ff4991d69e0c1 2024-11-13T10:39:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494449419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,433 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/5394d1bcd7a64430955ff4991d69e0c1, entries=150, sequenceid=93, filesize=11.7 K 2024-11-13T10:39:49,436 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0b3f34f131c1584553eed8127d585c45 in 977ms, sequenceid=93, compaction requested=true 2024-11-13T10:39:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-13T10:39:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-13T10:39:49,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-13T10:39:49,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7560 sec 2024-11-13T10:39:49,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.7670 sec 2024-11-13T10:39:49,451 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:49,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:39:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:49,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111305b060e6dc0646d391e1aef405303e43_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494389449/Put/seqid=0 2024-11-13T10:39:49,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742203_1379 (size=12154) 2024-11-13T10:39:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494449514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494449530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494449636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494449637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494449637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494449640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494449647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T10:39:49,838 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:49,838 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-13T10:39:49,841 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:49,842 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:49,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T10:39:49,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494449857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494449867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,919 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:49,925 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111305b060e6dc0646d391e1aef405303e43_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111305b060e6dc0646d391e1aef405303e43_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:49,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/7e7d2ddb73fe4325832950fc7706707f, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:49,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/7e7d2ddb73fe4325832950fc7706707f is 175, key is test_row_0/A:col10/1731494389449/Put/seqid=0 2024-11-13T10:39:49,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742204_1380 (size=30955) 2024-11-13T10:39:49,942 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/7e7d2ddb73fe4325832950fc7706707f 2024-11-13T10:39:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T10:39:49,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494449946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/19933c8ef2564d309414bb822d9fb1d1 is 50, key is test_row_0/B:col10/1731494389449/Put/seqid=0 2024-11-13T10:39:49,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742205_1381 (size=12001) 2024-11-13T10:39:49,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494449958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494449959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:49,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/19933c8ef2564d309414bb822d9fb1d1 2024-11-13T10:39:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:49,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:49,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/eb1a44749354420c85f3159df6281254 is 50, key is test_row_0/C:col10/1731494389449/Put/seqid=0 2024-11-13T10:39:49,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T10:39:49,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742206_1382 (size=12001) 2024-11-13T10:39:49,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:49,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:49,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:49,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:49,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:49,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/eb1a44749354420c85f3159df6281254 2024-11-13T10:39:50,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/7e7d2ddb73fe4325832950fc7706707f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7e7d2ddb73fe4325832950fc7706707f 2024-11-13T10:39:50,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7e7d2ddb73fe4325832950fc7706707f, entries=150, sequenceid=118, filesize=30.2 K 2024-11-13T10:39:50,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/19933c8ef2564d309414bb822d9fb1d1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/19933c8ef2564d309414bb822d9fb1d1 2024-11-13T10:39:50,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/19933c8ef2564d309414bb822d9fb1d1, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T10:39:50,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/eb1a44749354420c85f3159df6281254 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/eb1a44749354420c85f3159df6281254 2024-11-13T10:39:50,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/eb1a44749354420c85f3159df6281254, entries=150, sequenceid=118, filesize=11.7 K 2024-11-13T10:39:50,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0b3f34f131c1584553eed8127d585c45 in 575ms, sequenceid=118, compaction requested=true 2024-11-13T10:39:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:39:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T10:39:50,026 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:50,027 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:50,030 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:50,030 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/A is initiating minor compaction (all files) 2024-11-13T10:39:50,030 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:50,030 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/C is initiating minor compaction (all files) 2024-11-13T10:39:50,030 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/A in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:50,030 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/C in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:50,030 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b1606424455b4346bffe09897a5a6a4a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/287977efe18f48c3a83f6949c3d0f107, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/20726465897f4894834cdf8868e17da4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7e7d2ddb73fe4325832950fc7706707f] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=129.4 K 2024-11-13T10:39:50,030 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/2b8fb9f17a524d8d90aa67db31ff2fd4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/7737e125c6634f1584f6697aee4033ea, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/5394d1bcd7a64430955ff4991d69e0c1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/eb1a44749354420c85f3159df6281254] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=47.0 K 2024-11-13T10:39:50,030 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:50,030 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b1606424455b4346bffe09897a5a6a4a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/287977efe18f48c3a83f6949c3d0f107, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/20726465897f4894834cdf8868e17da4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7e7d2ddb73fe4325832950fc7706707f] 2024-11-13T10:39:50,031 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b8fb9f17a524d8d90aa67db31ff2fd4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494386857 2024-11-13T10:39:50,031 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1606424455b4346bffe09897a5a6a4a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494386857 2024-11-13T10:39:50,031 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7737e125c6634f1584f6697aee4033ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731494386946 2024-11-13T10:39:50,032 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 287977efe18f48c3a83f6949c3d0f107, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731494386946 2024-11-13T10:39:50,032 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5394d1bcd7a64430955ff4991d69e0c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731494386998 2024-11-13T10:39:50,032 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20726465897f4894834cdf8868e17da4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731494386998 2024-11-13T10:39:50,032 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting eb1a44749354420c85f3159df6281254, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731494389289 2024-11-13T10:39:50,032 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e7d2ddb73fe4325832950fc7706707f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731494389289 2024-11-13T10:39:50,067 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:50,085 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411132b1870e11f584bfd9d7d7890a938a4a3_0b3f34f131c1584553eed8127d585c45 store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:50,087 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:50,087 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/be829710bdab4f95822f6e30dc5e882a is 50, key is test_row_0/C:col10/1731494389449/Put/seqid=0 2024-11-13T10:39:50,088 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411132b1870e11f584bfd9d7d7890a938a4a3_0b3f34f131c1584553eed8127d585c45, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:50,088 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132b1870e11f584bfd9d7d7890a938a4a3_0b3f34f131c1584553eed8127d585c45 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:50,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742207_1383 (size=12241) 2024-11-13T10:39:50,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742208_1384 (size=4469) 2024-11-13T10:39:50,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T10:39:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:50,151 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:39:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T10:39:50,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130d098e0f65ee4d53add6f9d3be57b3b9_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494389510/Put/seqid=0 2024-11-13T10:39:50,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:50,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742209_1385 (size=12154) 2024-11-13T10:39:50,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:50,231 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130d098e0f65ee4d53add6f9d3be57b3b9_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130d098e0f65ee4d53add6f9d3be57b3b9_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:50,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/8a81a5496aaa4ef8bcf17335226b213d, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:50,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/8a81a5496aaa4ef8bcf17335226b213d is 175, key is test_row_0/A:col10/1731494389510/Put/seqid=0 2024-11-13T10:39:50,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742210_1386 (size=30955) 2024-11-13T10:39:50,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494450383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:50,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:50,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494450396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T10:39:50,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494450469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:50,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494450480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:50,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494450480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:50,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494450507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:50,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494450520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:50,545 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/be829710bdab4f95822f6e30dc5e882a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/be829710bdab4f95822f6e30dc5e882a 2024-11-13T10:39:50,551 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/C of 0b3f34f131c1584553eed8127d585c45 into be829710bdab4f95822f6e30dc5e882a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:50,551 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:50,552 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/C, priority=12, startTime=1731494390026; duration=0sec 2024-11-13T10:39:50,552 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:50,552 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:C 2024-11-13T10:39:50,552 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:50,554 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:50,554 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/B is initiating minor compaction (all files) 2024-11-13T10:39:50,554 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/B in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:50,554 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/bcdfbbff63ff42808daae1f3720308f6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/ef00af45f8fb4640b76d1b138e5b6f3a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/23cb9717b85b49b7bef321f117a90634, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/19933c8ef2564d309414bb822d9fb1d1] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=47.0 K 2024-11-13T10:39:50,555 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#A#compaction#327 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:50,555 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f5790f6e7b2c4f2697df69f0e919634c is 175, key is test_row_0/A:col10/1731494389449/Put/seqid=0 2024-11-13T10:39:50,557 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting bcdfbbff63ff42808daae1f3720308f6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731494386857 2024-11-13T10:39:50,557 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ef00af45f8fb4640b76d1b138e5b6f3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731494386946 2024-11-13T10:39:50,557 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 23cb9717b85b49b7bef321f117a90634, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731494386998 2024-11-13T10:39:50,558 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 19933c8ef2564d309414bb822d9fb1d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731494389289 2024-11-13T10:39:50,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742211_1387 (size=31195) 2024-11-13T10:39:50,581 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f5790f6e7b2c4f2697df69f0e919634c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f5790f6e7b2c4f2697df69f0e919634c 2024-11-13T10:39:50,582 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:50,582 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/c8e028a690a94adb92eede45876df54d is 50, key is test_row_0/B:col10/1731494389449/Put/seqid=0 2024-11-13T10:39:50,590 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/A of 0b3f34f131c1584553eed8127d585c45 into f5790f6e7b2c4f2697df69f0e919634c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:50,590 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:50,590 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/A, priority=12, startTime=1731494390026; duration=0sec 2024-11-13T10:39:50,592 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:50,592 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:A 2024-11-13T10:39:50,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742212_1388 (size=12241) 2024-11-13T10:39:50,614 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/c8e028a690a94adb92eede45876df54d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/c8e028a690a94adb92eede45876df54d 2024-11-13T10:39:50,632 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/B of 0b3f34f131c1584553eed8127d585c45 into c8e028a690a94adb92eede45876df54d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:50,632 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:50,632 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/B, priority=12, startTime=1731494390026; duration=0sec 2024-11-13T10:39:50,632 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:50,632 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:B 2024-11-13T10:39:50,643 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/8a81a5496aaa4ef8bcf17335226b213d 2024-11-13T10:39:50,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/73ed2270cf4144f7bc8f516d134565a8 is 50, key is test_row_0/B:col10/1731494389510/Put/seqid=0 2024-11-13T10:39:50,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742213_1389 (size=12001) 2024-11-13T10:39:50,696 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/73ed2270cf4144f7bc8f516d134565a8 2024-11-13T10:39:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494450727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:50,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/378ef630fc0547b090ff8a065cea7521 is 50, key is test_row_0/C:col10/1731494389510/Put/seqid=0 2024-11-13T10:39:50,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494450740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:50,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:50,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742214_1390 (size=12001) 2024-11-13T10:39:50,766 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/378ef630fc0547b090ff8a065cea7521 2024-11-13T10:39:50,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/8a81a5496aaa4ef8bcf17335226b213d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/8a81a5496aaa4ef8bcf17335226b213d 2024-11-13T10:39:50,791 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/8a81a5496aaa4ef8bcf17335226b213d, entries=150, sequenceid=129, filesize=30.2 K 2024-11-13T10:39:50,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/73ed2270cf4144f7bc8f516d134565a8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/73ed2270cf4144f7bc8f516d134565a8 2024-11-13T10:39:50,798 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/73ed2270cf4144f7bc8f516d134565a8, entries=150, sequenceid=129, filesize=11.7 K 2024-11-13T10:39:50,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/378ef630fc0547b090ff8a065cea7521 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/378ef630fc0547b090ff8a065cea7521 2024-11-13T10:39:50,803 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/378ef630fc0547b090ff8a065cea7521, entries=150, sequenceid=129, filesize=11.7 K 2024-11-13T10:39:50,809 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0b3f34f131c1584553eed8127d585c45 in 659ms, sequenceid=129, compaction requested=false 2024-11-13T10:39:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-13T10:39:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-13T10:39:50,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-13T10:39:50,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 977 msec 2024-11-13T10:39:50,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 985 msec 2024-11-13T10:39:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T10:39:50,981 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:50,982 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-13T10:39:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T10:39:50,987 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:50,988 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:50,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:51,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:39:51,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b2fe8f0107f647a29f376248c087420e_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494391050/Put/seqid=0 2024-11-13T10:39:51,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742215_1391 (size=14794) 2024-11-13T10:39:51,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494451083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T10:39:51,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494451087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T10:39:51,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:51,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494451196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494451207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T10:39:51,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:51,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T10:39:51,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494451407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494451421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T10:39:51,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:51,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,469 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:51,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b2fe8f0107f647a29f376248c087420e_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b2fe8f0107f647a29f376248c087420e_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:51,481 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/b3b989a77fc942d9af28aa136cf66f0d, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:51,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/b3b989a77fc942d9af28aa136cf66f0d is 175, key is test_row_0/A:col10/1731494391050/Put/seqid=0 2024-11-13T10:39:51,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742216_1392 (size=39749) 2024-11-13T10:39:51,504 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/b3b989a77fc942d9af28aa136cf66f0d 2024-11-13T10:39:51,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/e3ba396bfecb4affb1efe6c76b65ba20 is 50, key is test_row_0/B:col10/1731494391050/Put/seqid=0 2024-11-13T10:39:51,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494451508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494451517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,518 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_2' in 0b3f34f131c1584553eed8127d585c45 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:51,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494451517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,519 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in 0b3f34f131c1584553eed8127d585c45 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,519 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 0b3f34f131c1584553eed8127d585c45 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:51,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742217_1393 (size=12151) 2024-11-13T10:39:51,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T10:39:51,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:51,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T10:39:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494451728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494451737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:51,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:51,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T10:39:51,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T10:39:51,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:51,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:51,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:51,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/e3ba396bfecb4affb1efe6c76b65ba20 2024-11-13T10:39:51,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/9ab8368a62144265bcef6432894e2a63 is 50, key is test_row_0/C:col10/1731494391050/Put/seqid=0 2024-11-13T10:39:51,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742218_1394 (size=12151) 2024-11-13T10:39:52,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T10:39:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:52,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:52,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:52,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:52,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T10:39:52,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T10:39:52,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:52,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:52,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:52,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:52,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494452247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:52,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494452247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:52,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/9ab8368a62144265bcef6432894e2a63 2024-11-13T10:39:52,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/b3b989a77fc942d9af28aa136cf66f0d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b3b989a77fc942d9af28aa136cf66f0d 2024-11-13T10:39:52,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b3b989a77fc942d9af28aa136cf66f0d, entries=200, sequenceid=158, filesize=38.8 K 2024-11-13T10:39:52,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/e3ba396bfecb4affb1efe6c76b65ba20 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e3ba396bfecb4affb1efe6c76b65ba20 2024-11-13T10:39:52,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e3ba396bfecb4affb1efe6c76b65ba20, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T10:39:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/9ab8368a62144265bcef6432894e2a63 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/9ab8368a62144265bcef6432894e2a63 2024-11-13T10:39:52,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/9ab8368a62144265bcef6432894e2a63, entries=150, sequenceid=158, filesize=11.9 K 2024-11-13T10:39:52,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0b3f34f131c1584553eed8127d585c45 in 1323ms, sequenceid=158, compaction requested=true 2024-11-13T10:39:52,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:52,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:52,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:52,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:52,376 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:52,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:52,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:52,376 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:52,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:52,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T10:39:52,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:52,377 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T10:39:52,377 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:52,377 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/B is initiating minor compaction (all files) 2024-11-13T10:39:52,377 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/B in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:52,378 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/c8e028a690a94adb92eede45876df54d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/73ed2270cf4144f7bc8f516d134565a8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e3ba396bfecb4affb1efe6c76b65ba20] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=35.5 K 2024-11-13T10:39:52,378 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:52,378 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/A is initiating minor compaction (all files) 2024-11-13T10:39:52,378 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/A in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:52,378 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f5790f6e7b2c4f2697df69f0e919634c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/8a81a5496aaa4ef8bcf17335226b213d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b3b989a77fc942d9af28aa136cf66f0d] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=99.5 K 2024-11-13T10:39:52,378 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:52,378 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f5790f6e7b2c4f2697df69f0e919634c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/8a81a5496aaa4ef8bcf17335226b213d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b3b989a77fc942d9af28aa136cf66f0d] 2024-11-13T10:39:52,379 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting c8e028a690a94adb92eede45876df54d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731494389289 2024-11-13T10:39:52,379 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5790f6e7b2c4f2697df69f0e919634c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731494389289 2024-11-13T10:39:52,379 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ed2270cf4144f7bc8f516d134565a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731494389508 2024-11-13T10:39:52,380 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a81a5496aaa4ef8bcf17335226b213d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731494389508 2024-11-13T10:39:52,380 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e3ba396bfecb4affb1efe6c76b65ba20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731494390375 2024-11-13T10:39:52,380 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3b989a77fc942d9af28aa136cf66f0d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731494390375 2024-11-13T10:39:52,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111359cd000915ac46cdb250d916dea525ac_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494391078/Put/seqid=0 2024-11-13T10:39:52,394 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#B#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:52,395 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/6c39331105e54810a8943e5587084547 is 50, key is test_row_0/B:col10/1731494391050/Put/seqid=0 2024-11-13T10:39:52,396 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:52,403 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411139e971af27e194a8a8da57b6d0fdede86_0b3f34f131c1584553eed8127d585c45 store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:52,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742219_1395 (size=12304) 2024-11-13T10:39:52,404 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411139e971af27e194a8a8da57b6d0fdede86_0b3f34f131c1584553eed8127d585c45, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:52,404 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139e971af27e194a8a8da57b6d0fdede86_0b3f34f131c1584553eed8127d585c45 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:52,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742220_1396 (size=12493) 2024-11-13T10:39:52,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742221_1397 (size=4469) 2024-11-13T10:39:52,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:52,808 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111359cd000915ac46cdb250d916dea525ac_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111359cd000915ac46cdb250d916dea525ac_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/210d7ddb516c48ce9a40d521b8f1fd25, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:52,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/210d7ddb516c48ce9a40d521b8f1fd25 is 175, key is test_row_0/A:col10/1731494391078/Put/seqid=0 2024-11-13T10:39:52,820 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/6c39331105e54810a8943e5587084547 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6c39331105e54810a8943e5587084547 2024-11-13T10:39:52,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742222_1398 (size=31105) 2024-11-13T10:39:52,826 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/B of 0b3f34f131c1584553eed8127d585c45 into 6c39331105e54810a8943e5587084547(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:52,826 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:52,826 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/B, priority=13, startTime=1731494392376; duration=0sec 2024-11-13T10:39:52,826 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:52,826 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:B 2024-11-13T10:39:52,826 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:52,826 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#A#compaction#338 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:52,827 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/9714ae9fbd214ed9b4d14cfe6a8ead46 is 175, key is test_row_0/A:col10/1731494391050/Put/seqid=0 2024-11-13T10:39:52,827 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:52,827 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/C is initiating minor compaction (all files) 2024-11-13T10:39:52,827 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/C in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:52,827 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/be829710bdab4f95822f6e30dc5e882a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/378ef630fc0547b090ff8a065cea7521, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/9ab8368a62144265bcef6432894e2a63] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=35.5 K 2024-11-13T10:39:52,827 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting be829710bdab4f95822f6e30dc5e882a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731494389289 2024-11-13T10:39:52,828 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 378ef630fc0547b090ff8a065cea7521, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731494389508 2024-11-13T10:39:52,828 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ab8368a62144265bcef6432894e2a63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731494390375 2024-11-13T10:39:52,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742223_1399 (size=31447) 2024-11-13T10:39:52,836 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:52,837 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/9714ae9fbd214ed9b4d14cfe6a8ead46 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9714ae9fbd214ed9b4d14cfe6a8ead46 2024-11-13T10:39:52,837 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ce63c286b3d440518bdf2ffa2ce14f23 is 50, key is test_row_0/C:col10/1731494391050/Put/seqid=0 2024-11-13T10:39:52,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742224_1400 (size=12493) 2024-11-13T10:39:52,844 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/A of 0b3f34f131c1584553eed8127d585c45 into 9714ae9fbd214ed9b4d14cfe6a8ead46(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:52,844 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:52,844 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/A, priority=13, startTime=1731494392376; duration=0sec 2024-11-13T10:39:52,844 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:52,844 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:A 2024-11-13T10:39:52,849 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ce63c286b3d440518bdf2ffa2ce14f23 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ce63c286b3d440518bdf2ffa2ce14f23 2024-11-13T10:39:52,855 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/C of 0b3f34f131c1584553eed8127d585c45 into ce63c286b3d440518bdf2ffa2ce14f23(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:52,855 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:52,855 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/C, priority=13, startTime=1731494392376; duration=0sec 2024-11-13T10:39:52,855 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:52,855 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:C 2024-11-13T10:39:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T10:39:53,223 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/210d7ddb516c48ce9a40d521b8f1fd25 2024-11-13T10:39:53,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/69a4567296954bc8bc9ee23e9cb342ab is 50, key is test_row_0/B:col10/1731494391078/Put/seqid=0 2024-11-13T10:39:53,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742225_1401 (size=12151) 2024-11-13T10:39:53,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:53,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494453448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:53,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494453448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:53,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494453527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:53,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494453527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:53,532 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_2' in 0b3f34f131c1584553eed8127d585c45 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:53,532 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in 0b3f34f131c1584553eed8127d585c45 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:53,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494453547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:53,556 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 0b3f34f131c1584553eed8127d585c45 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:53,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494453567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:53,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494453567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:53,659 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/69a4567296954bc8bc9ee23e9cb342ab 2024-11-13T10:39:53,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/0d544ba1ae684f6bbfaf50033221de06 is 50, key is test_row_0/C:col10/1731494391078/Put/seqid=0 2024-11-13T10:39:53,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742226_1402 (size=12151) 2024-11-13T10:39:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494453787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494453787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:54,094 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/0d544ba1ae684f6bbfaf50033221de06 2024-11-13T10:39:54,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/210d7ddb516c48ce9a40d521b8f1fd25 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/210d7ddb516c48ce9a40d521b8f1fd25 2024-11-13T10:39:54,102 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/210d7ddb516c48ce9a40d521b8f1fd25, entries=150, sequenceid=168, filesize=30.4 K 2024-11-13T10:39:54,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/69a4567296954bc8bc9ee23e9cb342ab as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/69a4567296954bc8bc9ee23e9cb342ab 2024-11-13T10:39:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494454098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:54,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494454098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:54,109 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/69a4567296954bc8bc9ee23e9cb342ab, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T10:39:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/0d544ba1ae684f6bbfaf50033221de06 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/0d544ba1ae684f6bbfaf50033221de06 2024-11-13T10:39:54,113 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/0d544ba1ae684f6bbfaf50033221de06, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T10:39:54,114 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 0b3f34f131c1584553eed8127d585c45 in 1736ms, sequenceid=168, compaction requested=false 2024-11-13T10:39:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-13T10:39:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-13T10:39:54,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-13T10:39:54,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1270 sec 2024-11-13T10:39:54,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.1370 sec 2024-11-13T10:39:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:54,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-13T10:39:54,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134ab669bea23c415bbbe3588cc3387bfd_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494394619/Put/seqid=0 2024-11-13T10:39:54,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742227_1403 (size=14794) 2024-11-13T10:39:54,641 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,645 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134ab669bea23c415bbbe3588cc3387bfd_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134ab669bea23c415bbbe3588cc3387bfd_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:54,646 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/af16d0f7eab54aa28c68007cf912de92, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:54,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/af16d0f7eab54aa28c68007cf912de92 is 175, key is test_row_0/A:col10/1731494394619/Put/seqid=0 2024-11-13T10:39:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494454644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494454645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:54,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742228_1404 (size=39749) 2024-11-13T10:39:54,659 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/af16d0f7eab54aa28c68007cf912de92 2024-11-13T10:39:54,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/718bfab91dbb44a7ba7e4725e4c21930 is 50, key is test_row_0/B:col10/1731494394619/Put/seqid=0 2024-11-13T10:39:54,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742229_1405 (size=12151) 2024-11-13T10:39:54,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/718bfab91dbb44a7ba7e4725e4c21930 2024-11-13T10:39:54,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/6b187854e32c4ce890879ebcddfcf2de is 50, key is test_row_0/C:col10/1731494394619/Put/seqid=0 2024-11-13T10:39:54,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742230_1406 (size=12151) 2024-11-13T10:39:54,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/6b187854e32c4ce890879ebcddfcf2de 2024-11-13T10:39:54,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/af16d0f7eab54aa28c68007cf912de92 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/af16d0f7eab54aa28c68007cf912de92 2024-11-13T10:39:54,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/af16d0f7eab54aa28c68007cf912de92, entries=200, sequenceid=199, filesize=38.8 K 2024-11-13T10:39:54,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/718bfab91dbb44a7ba7e4725e4c21930 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/718bfab91dbb44a7ba7e4725e4c21930 2024-11-13T10:39:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/718bfab91dbb44a7ba7e4725e4c21930, entries=150, sequenceid=199, filesize=11.9 K 2024-11-13T10:39:54,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/6b187854e32c4ce890879ebcddfcf2de as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6b187854e32c4ce890879ebcddfcf2de 2024-11-13T10:39:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6b187854e32c4ce890879ebcddfcf2de, entries=150, sequenceid=199, filesize=11.9 K 2024-11-13T10:39:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 0b3f34f131c1584553eed8127d585c45 in 100ms, sequenceid=199, compaction requested=true 2024-11-13T10:39:54,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,721 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:54,722 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,723 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:54,723 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/A is initiating minor compaction (all files) 2024-11-13T10:39:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,723 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/A in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:54,723 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9714ae9fbd214ed9b4d14cfe6a8ead46, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/210d7ddb516c48ce9a40d521b8f1fd25, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/af16d0f7eab54aa28c68007cf912de92] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=99.9 K 2024-11-13T10:39:54,723 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:54,723 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9714ae9fbd214ed9b4d14cfe6a8ead46, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/210d7ddb516c48ce9a40d521b8f1fd25, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/af16d0f7eab54aa28c68007cf912de92] 2024-11-13T10:39:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:54,724 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:54,724 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/B is initiating minor compaction (all files) 2024-11-13T10:39:54,724 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/B in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:54,724 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9714ae9fbd214ed9b4d14cfe6a8ead46, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731494390375 2024-11-13T10:39:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,724 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6c39331105e54810a8943e5587084547, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/69a4567296954bc8bc9ee23e9cb342ab, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/718bfab91dbb44a7ba7e4725e4c21930] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=35.9 K 2024-11-13T10:39:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,724 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 210d7ddb516c48ce9a40d521b8f1fd25, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731494391071 2024-11-13T10:39:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,724 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c39331105e54810a8943e5587084547, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731494390375 2024-11-13T10:39:54,724 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting af16d0f7eab54aa28c68007cf912de92, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731494393391 2024-11-13T10:39:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,725 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 69a4567296954bc8bc9ee23e9cb342ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731494391071 2024-11-13T10:39:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,725 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 718bfab91dbb44a7ba7e4725e4c21930, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731494393391 2024-11-13T10:39:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,731 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,734 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113dd3f8a9baa2f467298be94774e63266d_0b3f34f131c1584553eed8127d585c45 store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,735 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#B#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:54,735 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113dd3f8a9baa2f467298be94774e63266d_0b3f34f131c1584553eed8127d585c45, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,736 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dd3f8a9baa2f467298be94774e63266d_0b3f34f131c1584553eed8127d585c45 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:54,736 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/060d1644e5364b6d8ffee42c7a81f19d is 50, key is test_row_0/B:col10/1731494394619/Put/seqid=0 2024-11-13T10:39:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742231_1407 (size=4469) 2024-11-13T10:39:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742232_1408 (size=12595) 2024-11-13T10:39:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,748 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/060d1644e5364b6d8ffee42c7a81f19d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060d1644e5364b6d8ffee42c7a81f19d 2024-11-13T10:39:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,765 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/B of 0b3f34f131c1584553eed8127d585c45 into 060d1644e5364b6d8ffee42c7a81f19d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:54,765 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,765 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/B, priority=13, startTime=1731494394722; duration=0sec 2024-11-13T10:39:54,765 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:54,765 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:B 2024-11-13T10:39:54,766 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:39:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,766 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:39:54,767 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/C is initiating minor compaction (all files) 2024-11-13T10:39:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,767 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/C in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,768 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ce63c286b3d440518bdf2ffa2ce14f23, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/0d544ba1ae684f6bbfaf50033221de06, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6b187854e32c4ce890879ebcddfcf2de] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=35.9 K 2024-11-13T10:39:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,768 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ce63c286b3d440518bdf2ffa2ce14f23, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731494390375 2024-11-13T10:39:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,769 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d544ba1ae684f6bbfaf50033221de06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731494391071 2024-11-13T10:39:54,769 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b187854e32c4ce890879ebcddfcf2de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731494393391 2024-11-13T10:39:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:54,784 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:54,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:39:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,786 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,787 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/009d70f8fe4b4c3494c4d50937b9bcae is 50, key is test_row_0/C:col10/1731494394619/Put/seqid=0 2024-11-13T10:39:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fbb335d48ddd4b79b9e2ab68220e110c_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494394782/Put/seqid=0 2024-11-13T10:39:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742233_1409 (size=12595) 2024-11-13T10:39:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,820 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/009d70f8fe4b4c3494c4d50937b9bcae as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/009d70f8fe4b4c3494c4d50937b9bcae 2024-11-13T10:39:54,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742234_1410 (size=17284) 2024-11-13T10:39:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:54,827 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/C of 0b3f34f131c1584553eed8127d585c45 into 009d70f8fe4b4c3494c4d50937b9bcae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:54,827 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:54,827 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/C, priority=13, startTime=1731494394724; duration=0sec 2024-11-13T10:39:54,827 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:54,827 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:C 2024-11-13T10:39:54,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494454948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494454949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:55,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494455070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:55,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494455070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:55,142 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#A#compaction#345 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:55,143 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/a8ffc71e0d97477180b9204a56b0f5f5 is 175, key is test_row_0/A:col10/1731494394619/Put/seqid=0 2024-11-13T10:39:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T10:39:55,158 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:55,161 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-13T10:39:55,163 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T10:39:55,165 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:55,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:55,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742235_1411 (size=31549) 2024-11-13T10:39:55,175 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/a8ffc71e0d97477180b9204a56b0f5f5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/a8ffc71e0d97477180b9204a56b0f5f5 2024-11-13T10:39:55,179 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/A of 0b3f34f131c1584553eed8127d585c45 into a8ffc71e0d97477180b9204a56b0f5f5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:55,179 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:55,179 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/A, priority=13, startTime=1731494394721; duration=0sec 2024-11-13T10:39:55,179 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:55,179 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:A 2024-11-13T10:39:55,221 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:55,224 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fbb335d48ddd4b79b9e2ab68220e110c_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fbb335d48ddd4b79b9e2ab68220e110c_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:55,225 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d217c9e2bd274c978d52f8929a28bf0f, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:55,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d217c9e2bd274c978d52f8929a28bf0f is 175, key is test_row_0/A:col10/1731494394782/Put/seqid=0 2024-11-13T10:39:55,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742236_1412 (size=48389) 2024-11-13T10:39:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T10:39:55,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494455286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:55,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494455287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:55,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T10:39:55,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:55,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:55,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:55,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:55,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:55,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T10:39:55,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:55,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:55,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:55,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:55,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T10:39:55,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494455597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:55,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494455597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:55,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:55,630 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d217c9e2bd274c978d52f8929a28bf0f 2024-11-13T10:39:55,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T10:39:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:55,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:55,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/4d40c30a0dae478c950fd25fea64402d is 50, key is test_row_0/B:col10/1731494394782/Put/seqid=0 2024-11-13T10:39:55,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742237_1413 (size=12151) 2024-11-13T10:39:55,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/4d40c30a0dae478c950fd25fea64402d 2024-11-13T10:39:55,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/d6bc9ec6455a496588f430f9828cbd5c is 50, key is test_row_0/C:col10/1731494394782/Put/seqid=0 2024-11-13T10:39:55,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742238_1414 (size=12151) 2024-11-13T10:39:55,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/d6bc9ec6455a496588f430f9828cbd5c 2024-11-13T10:39:55,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d217c9e2bd274c978d52f8929a28bf0f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d217c9e2bd274c978d52f8929a28bf0f 2024-11-13T10:39:55,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d217c9e2bd274c978d52f8929a28bf0f, entries=250, sequenceid=211, filesize=47.3 K 2024-11-13T10:39:55,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/4d40c30a0dae478c950fd25fea64402d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/4d40c30a0dae478c950fd25fea64402d 2024-11-13T10:39:55,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/4d40c30a0dae478c950fd25fea64402d, entries=150, sequenceid=211, filesize=11.9 K 2024-11-13T10:39:55,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/d6bc9ec6455a496588f430f9828cbd5c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/d6bc9ec6455a496588f430f9828cbd5c 2024-11-13T10:39:55,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/d6bc9ec6455a496588f430f9828cbd5c, entries=150, sequenceid=211, filesize=11.9 K 2024-11-13T10:39:55,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0b3f34f131c1584553eed8127d585c45 in 971ms, sequenceid=211, compaction requested=false 2024-11-13T10:39:55,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:55,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T10:39:55,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:55,789 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:39:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T10:39:55,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111331ce4e872e974f0286ebef33ce8811c3_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494394927/Put/seqid=0 2024-11-13T10:39:55,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742239_1415 (size=12304) 2024-11-13T10:39:55,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:55,843 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111331ce4e872e974f0286ebef33ce8811c3_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111331ce4e872e974f0286ebef33ce8811c3_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:55,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d487cc65fdad4c21a33a1bcf824ff525, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:55,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d487cc65fdad4c21a33a1bcf824ff525 is 175, key is test_row_0/A:col10/1731494394927/Put/seqid=0 2024-11-13T10:39:55,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742240_1416 (size=31105) 2024-11-13T10:39:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:56,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:56,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494456163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:56,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494456164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:56,250 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d487cc65fdad4c21a33a1bcf824ff525 2024-11-13T10:39:56,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/04fb3eb3668a461c8bec35e6bba5d4bd is 50, key is test_row_0/B:col10/1731494394927/Put/seqid=0 2024-11-13T10:39:56,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742241_1417 (size=12151) 2024-11-13T10:39:56,280 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/04fb3eb3668a461c8bec35e6bba5d4bd 2024-11-13T10:39:56,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494456277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:56,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/747d90bce72448939cebe7361e861f57 is 50, key is test_row_0/C:col10/1731494394927/Put/seqid=0 2024-11-13T10:39:56,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742242_1418 (size=12151) 2024-11-13T10:39:56,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494456288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T10:39:56,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494456499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:56,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494456507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:56,692 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/747d90bce72448939cebe7361e861f57 2024-11-13T10:39:56,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d487cc65fdad4c21a33a1bcf824ff525 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d487cc65fdad4c21a33a1bcf824ff525 2024-11-13T10:39:56,704 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d487cc65fdad4c21a33a1bcf824ff525, entries=150, sequenceid=238, filesize=30.4 K 2024-11-13T10:39:56,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/04fb3eb3668a461c8bec35e6bba5d4bd as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/04fb3eb3668a461c8bec35e6bba5d4bd 2024-11-13T10:39:56,709 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/04fb3eb3668a461c8bec35e6bba5d4bd, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T10:39:56,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/747d90bce72448939cebe7361e861f57 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/747d90bce72448939cebe7361e861f57 2024-11-13T10:39:56,732 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/747d90bce72448939cebe7361e861f57, entries=150, sequenceid=238, filesize=11.9 K 2024-11-13T10:39:56,733 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0b3f34f131c1584553eed8127d585c45 in 944ms, sequenceid=238, compaction requested=true 2024-11-13T10:39:56,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:56,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:56,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-13T10:39:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-13T10:39:56,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-13T10:39:56,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-13T10:39:56,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.5760 sec 2024-11-13T10:39:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:56,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T10:39:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:56,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ebf692514dba4079a21555ac1f11da3c_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494396162/Put/seqid=0 2024-11-13T10:39:56,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742243_1419 (size=14794) 2024-11-13T10:39:56,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494456963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:56,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494456965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:56,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:56,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:57,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494457087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:57,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494457087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:57,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:57,290 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ebf692514dba4079a21555ac1f11da3c_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ebf692514dba4079a21555ac1f11da3c_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:57,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f2652d16e63849d48cf2235eddee15de, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:57,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f2652d16e63849d48cf2235eddee15de is 175, key is test_row_0/A:col10/1731494396162/Put/seqid=0 2024-11-13T10:39:57,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742244_1420 (size=39749) 2024-11-13T10:39:57,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494457307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494457307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T10:39:57,322 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:39:57,322 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:39:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-13T10:39:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T10:39:57,325 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:39:57,325 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:39:57,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:39:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T10:39:57,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T10:39:57,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:57,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:57,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:57,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57504 deadline: 1731494457557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:57,568 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_2' in 0b3f34f131c1584553eed8127d585c45 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:57,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57462 deadline: 1731494457580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:57,585 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in 0b3f34f131c1584553eed8127d585c45 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8294 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:57,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57464 deadline: 1731494457598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:57,606 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in 0b3f34f131c1584553eed8127d585c45 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8316 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:39:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:57,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T10:39:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:57,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494457631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:57,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494457632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T10:39:57,711 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f2652d16e63849d48cf2235eddee15de 2024-11-13T10:39:57,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/061f4df175284a49b5f4f302adcbe623 is 50, key is test_row_0/B:col10/1731494396162/Put/seqid=0 2024-11-13T10:39:57,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742245_1421 (size=12151) 2024-11-13T10:39:57,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T10:39:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:57,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T10:39:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:57,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T10:39:58,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T10:39:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:58,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:58,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/061f4df175284a49b5f4f302adcbe623 2024-11-13T10:39:58,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/c164655c97dc46cf995dd57d5b8f9e7b is 50, key is test_row_0/C:col10/1731494396162/Put/seqid=0 2024-11-13T10:39:58,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494458149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:58,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494458149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:58,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742246_1422 (size=12151) 2024-11-13T10:39:58,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/c164655c97dc46cf995dd57d5b8f9e7b 2024-11-13T10:39:58,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f2652d16e63849d48cf2235eddee15de as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f2652d16e63849d48cf2235eddee15de 2024-11-13T10:39:58,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f2652d16e63849d48cf2235eddee15de, entries=200, sequenceid=250, filesize=38.8 K 2024-11-13T10:39:58,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/061f4df175284a49b5f4f302adcbe623 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/061f4df175284a49b5f4f302adcbe623 2024-11-13T10:39:58,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/061f4df175284a49b5f4f302adcbe623, entries=150, sequenceid=250, filesize=11.9 K 2024-11-13T10:39:58,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/c164655c97dc46cf995dd57d5b8f9e7b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/c164655c97dc46cf995dd57d5b8f9e7b 2024-11-13T10:39:58,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T10:39:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:58,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:39:58,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/c164655c97dc46cf995dd57d5b8f9e7b, entries=150, sequenceid=250, filesize=11.9 K 2024-11-13T10:39:58,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0b3f34f131c1584553eed8127d585c45 in 1469ms, sequenceid=250, compaction requested=true 2024-11-13T10:39:58,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:58,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:39:58,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:58,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:39:58,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:58,293 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:58,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:39:58,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:39:58,293 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:58,297 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150792 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:58,297 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/A is initiating minor compaction (all files) 2024-11-13T10:39:58,298 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/A in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:58,298 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/a8ffc71e0d97477180b9204a56b0f5f5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d217c9e2bd274c978d52f8929a28bf0f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d487cc65fdad4c21a33a1bcf824ff525, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f2652d16e63849d48cf2235eddee15de] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=147.3 K 2024-11-13T10:39:58,298 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:58,298 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:58,298 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/a8ffc71e0d97477180b9204a56b0f5f5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d217c9e2bd274c978d52f8929a28bf0f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d487cc65fdad4c21a33a1bcf824ff525, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f2652d16e63849d48cf2235eddee15de] 2024-11-13T10:39:58,298 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/B is initiating minor compaction (all files) 2024-11-13T10:39:58,298 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/B in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:58,298 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060d1644e5364b6d8ffee42c7a81f19d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/4d40c30a0dae478c950fd25fea64402d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/04fb3eb3668a461c8bec35e6bba5d4bd, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/061f4df175284a49b5f4f302adcbe623] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=47.9 K 2024-11-13T10:39:58,298 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8ffc71e0d97477180b9204a56b0f5f5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731494393391 2024-11-13T10:39:58,298 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 060d1644e5364b6d8ffee42c7a81f19d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731494393391 2024-11-13T10:39:58,299 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d217c9e2bd274c978d52f8929a28bf0f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731494394633 2024-11-13T10:39:58,299 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d487cc65fdad4c21a33a1bcf824ff525, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731494394927 2024-11-13T10:39:58,299 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d40c30a0dae478c950fd25fea64402d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731494394633 2024-11-13T10:39:58,299 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2652d16e63849d48cf2235eddee15de, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494396134 2024-11-13T10:39:58,300 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 04fb3eb3668a461c8bec35e6bba5d4bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731494394927 2024-11-13T10:39:58,300 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 061f4df175284a49b5f4f302adcbe623, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494396144 2024-11-13T10:39:58,325 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:58,330 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#B#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:58,331 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/cb046275ae7e4bd8ba9e4d4f6d540993 is 50, key is test_row_0/B:col10/1731494396162/Put/seqid=0 2024-11-13T10:39:58,339 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113cdb118ff3c534ba5b2c936e5410998d7_0b3f34f131c1584553eed8127d585c45 store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:58,342 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113cdb118ff3c534ba5b2c936e5410998d7_0b3f34f131c1584553eed8127d585c45, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:58,342 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cdb118ff3c534ba5b2c936e5410998d7_0b3f34f131c1584553eed8127d585c45 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:58,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742247_1423 (size=12731) 2024-11-13T10:39:58,399 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/cb046275ae7e4bd8ba9e4d4f6d540993 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/cb046275ae7e4bd8ba9e4d4f6d540993 2024-11-13T10:39:58,404 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/B of 0b3f34f131c1584553eed8127d585c45 into cb046275ae7e4bd8ba9e4d4f6d540993(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:58,404 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:58,404 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/B, priority=12, startTime=1731494398292; duration=0sec 2024-11-13T10:39:58,404 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:39:58,404 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:B 2024-11-13T10:39:58,404 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:39:58,405 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:39:58,405 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/C is initiating minor compaction (all files) 2024-11-13T10:39:58,405 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/C in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:58,405 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/009d70f8fe4b4c3494c4d50937b9bcae, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/d6bc9ec6455a496588f430f9828cbd5c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/747d90bce72448939cebe7361e861f57, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/c164655c97dc46cf995dd57d5b8f9e7b] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=47.9 K 2024-11-13T10:39:58,406 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 009d70f8fe4b4c3494c4d50937b9bcae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731494393391 2024-11-13T10:39:58,406 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting d6bc9ec6455a496588f430f9828cbd5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731494394633 2024-11-13T10:39:58,406 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 747d90bce72448939cebe7361e861f57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731494394927 2024-11-13T10:39:58,407 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting c164655c97dc46cf995dd57d5b8f9e7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494396144 2024-11-13T10:39:58,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742248_1424 (size=4469) 2024-11-13T10:39:58,422 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#C#compaction#359 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:58,423 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/a3d73c7dea1d4fe480e548d350c2746f is 50, key is test_row_0/C:col10/1731494396162/Put/seqid=0 2024-11-13T10:39:58,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T10:39:58,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:58,439 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T10:39:58,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742249_1425 (size=12731) 2024-11-13T10:39:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T10:39:58,474 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/a3d73c7dea1d4fe480e548d350c2746f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a3d73c7dea1d4fe480e548d350c2746f 2024-11-13T10:39:58,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134186331ecff243559145e9af2a0e348b_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494396910/Put/seqid=0 2024-11-13T10:39:58,486 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/C of 0b3f34f131c1584553eed8127d585c45 into a3d73c7dea1d4fe480e548d350c2746f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:58,486 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:58,486 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/C, priority=12, startTime=1731494398293; duration=0sec 2024-11-13T10:39:58,486 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:58,486 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:C 2024-11-13T10:39:58,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742250_1426 (size=12454) 2024-11-13T10:39:58,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:58,522 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134186331ecff243559145e9af2a0e348b_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134186331ecff243559145e9af2a0e348b_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/ad3bb10044ab4680bd3ba07cb088551f, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:58,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/ad3bb10044ab4680bd3ba07cb088551f is 175, key is test_row_0/A:col10/1731494396910/Put/seqid=0 2024-11-13T10:39:58,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742251_1427 (size=31255) 2024-11-13T10:39:58,811 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#A#compaction#357 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:39:58,812 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/55a8751b58784a0682182b287b1af9f7 is 175, key is test_row_0/A:col10/1731494396162/Put/seqid=0 2024-11-13T10:39:58,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742252_1428 (size=31685) 2024-11-13T10:39:58,825 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/55a8751b58784a0682182b287b1af9f7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/55a8751b58784a0682182b287b1af9f7 2024-11-13T10:39:58,829 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/A of 0b3f34f131c1584553eed8127d585c45 into 55a8751b58784a0682182b287b1af9f7(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:39:58,829 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:58,829 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/A, priority=12, startTime=1731494398292; duration=0sec 2024-11-13T10:39:58,830 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:39:58,830 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:A 2024-11-13T10:39:58,936 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/ad3bb10044ab4680bd3ba07cb088551f 2024-11-13T10:39:58,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/e56ca6d0395540f2aa93f20e81be520a is 50, key is test_row_0/B:col10/1731494396910/Put/seqid=0 2024-11-13T10:39:58,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742253_1429 (size=12301) 2024-11-13T10:39:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:59,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:39:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494459237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:59,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494459254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:59,357 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/e56ca6d0395540f2aa93f20e81be520a 2024-11-13T10:39:59,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494459358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/55b674a271344bb2b21e2d543d0daf1a is 50, key is test_row_0/C:col10/1731494396910/Put/seqid=0 2024-11-13T10:39:59,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494459378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:59,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742254_1430 (size=12301) 2024-11-13T10:39:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T10:39:59,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494459567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:59,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:59,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:39:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494459606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:39:59,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:39:59,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:39:59,830 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/55b674a271344bb2b21e2d543d0daf1a 2024-11-13T10:39:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/ad3bb10044ab4680bd3ba07cb088551f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/ad3bb10044ab4680bd3ba07cb088551f 2024-11-13T10:39:59,850 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/ad3bb10044ab4680bd3ba07cb088551f, entries=150, sequenceid=275, filesize=30.5 K 2024-11-13T10:39:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/e56ca6d0395540f2aa93f20e81be520a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e56ca6d0395540f2aa93f20e81be520a 2024-11-13T10:39:59,859 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e56ca6d0395540f2aa93f20e81be520a, entries=150, sequenceid=275, filesize=12.0 K 2024-11-13T10:39:59,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/55b674a271344bb2b21e2d543d0daf1a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/55b674a271344bb2b21e2d543d0daf1a 2024-11-13T10:39:59,867 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/55b674a271344bb2b21e2d543d0daf1a, entries=150, sequenceid=275, filesize=12.0 K 2024-11-13T10:39:59,868 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0b3f34f131c1584553eed8127d585c45 in 1429ms, sequenceid=275, compaction requested=false 2024-11-13T10:39:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:39:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:39:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-13T10:39:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-13T10:39:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:59,888 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:39:59,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T10:39:59,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-13T10:39:59,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5500 sec 2024-11-13T10:39:59,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.5680 sec 2024-11-13T10:39:59,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111309c45c2c1ccc4e22bb2caf6d87c89d3e_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494399195/Put/seqid=0 2024-11-13T10:39:59,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742255_1431 (size=14994) 2024-11-13T10:39:59,973 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:39:59,977 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111309c45c2c1ccc4e22bb2caf6d87c89d3e_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111309c45c2c1ccc4e22bb2caf6d87c89d3e_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:39:59,978 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/7b0315e25af842269cd4d7de09680f06, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:39:59,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/7b0315e25af842269cd4d7de09680f06 is 175, key is test_row_0/A:col10/1731494399195/Put/seqid=0 2024-11-13T10:39:59,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742256_1432 (size=39949) 2024-11-13T10:39:59,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/7b0315e25af842269cd4d7de09680f06 2024-11-13T10:39:59,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/47f0de69b50f4cf7a2ed00687385684d is 50, key is test_row_0/B:col10/1731494399195/Put/seqid=0 2024-11-13T10:40:00,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742257_1433 (size=12301) 2024-11-13T10:40:00,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/47f0de69b50f4cf7a2ed00687385684d 2024-11-13T10:40:00,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/6e892582b4784b09be7419a0fd60c739 is 50, key is test_row_0/C:col10/1731494399195/Put/seqid=0 2024-11-13T10:40:00,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494460029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:00,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742258_1434 (size=12301) 2024-11-13T10:40:00,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/6e892582b4784b09be7419a0fd60c739 2024-11-13T10:40:00,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494460037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:00,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/7b0315e25af842269cd4d7de09680f06 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7b0315e25af842269cd4d7de09680f06 2024-11-13T10:40:00,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7b0315e25af842269cd4d7de09680f06, entries=200, sequenceid=290, filesize=39.0 K 2024-11-13T10:40:00,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/47f0de69b50f4cf7a2ed00687385684d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/47f0de69b50f4cf7a2ed00687385684d 2024-11-13T10:40:00,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/47f0de69b50f4cf7a2ed00687385684d, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T10:40:00,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/6e892582b4784b09be7419a0fd60c739 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6e892582b4784b09be7419a0fd60c739 2024-11-13T10:40:00,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6e892582b4784b09be7419a0fd60c739, entries=150, sequenceid=290, filesize=12.0 K 2024-11-13T10:40:00,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0b3f34f131c1584553eed8127d585c45 in 176ms, sequenceid=290, compaction requested=true 2024-11-13T10:40:00,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:00,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:00,065 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:00,065 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:00,065 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:00,065 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/A is initiating minor compaction (all files) 2024-11-13T10:40:00,065 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/A in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:00,065 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:00,066 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/55a8751b58784a0682182b287b1af9f7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/ad3bb10044ab4680bd3ba07cb088551f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7b0315e25af842269cd4d7de09680f06] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=100.5 K 2024-11-13T10:40:00,066 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/B is initiating minor compaction (all files) 2024-11-13T10:40:00,066 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:00,066 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/55a8751b58784a0682182b287b1af9f7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/ad3bb10044ab4680bd3ba07cb088551f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7b0315e25af842269cd4d7de09680f06] 2024-11-13T10:40:00,066 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/B in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:00,066 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/cb046275ae7e4bd8ba9e4d4f6d540993, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e56ca6d0395540f2aa93f20e81be520a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/47f0de69b50f4cf7a2ed00687385684d] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=36.5 K 2024-11-13T10:40:00,066 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55a8751b58784a0682182b287b1af9f7, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494396144 2024-11-13T10:40:00,066 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting cb046275ae7e4bd8ba9e4d4f6d540993, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494396144 2024-11-13T10:40:00,066 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad3bb10044ab4680bd3ba07cb088551f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731494396910 2024-11-13T10:40:00,067 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0315e25af842269cd4d7de09680f06, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731494399195 2024-11-13T10:40:00,068 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e56ca6d0395540f2aa93f20e81be520a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731494396910 2024-11-13T10:40:00,068 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f0de69b50f4cf7a2ed00687385684d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731494399195 2024-11-13T10:40:00,073 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:00,074 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111336ee3378c49f4a64b482ead492c3509f_0b3f34f131c1584553eed8127d585c45 store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:00,075 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#B#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:00,076 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/38d1bcab325a422b964a75131e3b8f80 is 50, key is test_row_0/B:col10/1731494399195/Put/seqid=0 2024-11-13T10:40:00,076 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111336ee3378c49f4a64b482ead492c3509f_0b3f34f131c1584553eed8127d585c45, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:00,076 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111336ee3378c49f4a64b482ead492c3509f_0b3f34f131c1584553eed8127d585c45 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:00,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742259_1435 (size=4469) 2024-11-13T10:40:00,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742260_1436 (size=12983) 2024-11-13T10:40:00,105 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/38d1bcab325a422b964a75131e3b8f80 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/38d1bcab325a422b964a75131e3b8f80 2024-11-13T10:40:00,111 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/B of 0b3f34f131c1584553eed8127d585c45 into 38d1bcab325a422b964a75131e3b8f80(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:00,111 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:00,111 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/B, priority=13, startTime=1731494400065; duration=0sec 2024-11-13T10:40:00,111 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:00,111 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:B 2024-11-13T10:40:00,111 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:00,113 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:00,113 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/C is initiating minor compaction (all files) 2024-11-13T10:40:00,113 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/C in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:00,114 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a3d73c7dea1d4fe480e548d350c2746f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/55b674a271344bb2b21e2d543d0daf1a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6e892582b4784b09be7419a0fd60c739] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=36.5 K 2024-11-13T10:40:00,114 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting a3d73c7dea1d4fe480e548d350c2746f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494396144 2024-11-13T10:40:00,114 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 55b674a271344bb2b21e2d543d0daf1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731494396910 2024-11-13T10:40:00,115 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e892582b4784b09be7419a0fd60c739, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731494399195 2024-11-13T10:40:00,122 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#C#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:00,122 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/fdc28452195f4800b8f579dce53f649d is 50, key is test_row_0/C:col10/1731494399195/Put/seqid=0 2024-11-13T10:40:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:00,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T10:40:00,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742261_1437 (size=12983) 2024-11-13T10:40:00,177 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/fdc28452195f4800b8f579dce53f649d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/fdc28452195f4800b8f579dce53f649d 2024-11-13T10:40:00,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a4a5b97864fa40f6b1a99799f18cb0d3_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494400035/Put/seqid=0 2024-11-13T10:40:00,197 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/C of 0b3f34f131c1584553eed8127d585c45 into fdc28452195f4800b8f579dce53f649d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:00,197 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:00,197 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/C, priority=13, startTime=1731494400065; duration=0sec 2024-11-13T10:40:00,197 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:00,197 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:C 2024-11-13T10:40:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494460209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:00,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494460225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:00,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:00,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742262_1438 (size=14994) 2024-11-13T10:40:00,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494460337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:00,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494460346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:00,499 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#A#compaction#366 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:00,500 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/9d024fcfc49c4792b11913e804cba350 is 175, key is test_row_0/A:col10/1731494399195/Put/seqid=0 2024-11-13T10:40:00,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742263_1439 (size=31937) 2024-11-13T10:40:00,511 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/9d024fcfc49c4792b11913e804cba350 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9d024fcfc49c4792b11913e804cba350 2024-11-13T10:40:00,516 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/A of 0b3f34f131c1584553eed8127d585c45 into 9d024fcfc49c4792b11913e804cba350(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:00,516 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:00,516 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/A, priority=13, startTime=1731494400064; duration=0sec 2024-11-13T10:40:00,516 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:00,516 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:A 2024-11-13T10:40:00,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494460547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:00,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:00,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494460557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:00,646 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:00,655 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a4a5b97864fa40f6b1a99799f18cb0d3_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4a5b97864fa40f6b1a99799f18cb0d3_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:00,656 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/c9c0cfa4e11d4b2189888cbc41a9e72f, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:00,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/c9c0cfa4e11d4b2189888cbc41a9e72f is 175, key is test_row_0/A:col10/1731494400035/Put/seqid=0 2024-11-13T10:40:00,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742264_1440 (size=39949) 2024-11-13T10:40:00,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/c9c0cfa4e11d4b2189888cbc41a9e72f 2024-11-13T10:40:00,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/14a03196a1594c3cb566aa680b371dc9 is 50, key is test_row_0/B:col10/1731494400035/Put/seqid=0 2024-11-13T10:40:00,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742265_1441 (size=12301) 2024-11-13T10:40:00,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/14a03196a1594c3cb566aa680b371dc9 2024-11-13T10:40:00,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/f3d9478196fe4a79becb117250b5f83f is 50, key is test_row_0/C:col10/1731494400035/Put/seqid=0 2024-11-13T10:40:00,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742266_1442 (size=12301) 2024-11-13T10:40:00,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/f3d9478196fe4a79becb117250b5f83f 2024-11-13T10:40:00,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/c9c0cfa4e11d4b2189888cbc41a9e72f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/c9c0cfa4e11d4b2189888cbc41a9e72f 2024-11-13T10:40:00,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/c9c0cfa4e11d4b2189888cbc41a9e72f, entries=200, sequenceid=315, filesize=39.0 K 2024-11-13T10:40:00,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/14a03196a1594c3cb566aa680b371dc9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/14a03196a1594c3cb566aa680b371dc9 2024-11-13T10:40:00,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/14a03196a1594c3cb566aa680b371dc9, entries=150, sequenceid=315, filesize=12.0 K 2024-11-13T10:40:00,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/f3d9478196fe4a79becb117250b5f83f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/f3d9478196fe4a79becb117250b5f83f 2024-11-13T10:40:00,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/f3d9478196fe4a79becb117250b5f83f, entries=150, sequenceid=315, filesize=12.0 K 2024-11-13T10:40:00,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0b3f34f131c1584553eed8127d585c45 in 582ms, sequenceid=315, compaction requested=false 2024-11-13T10:40:00,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:00,873 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:00,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T10:40:00,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f77d67d4ff5a489bb0ddfed9720a8caf_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494400873/Put/seqid=0 2024-11-13T10:40:00,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742267_1443 (size=17534) 2024-11-13T10:40:00,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494460978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:00,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494460986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:01,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494461100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:01,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494461108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:01,305 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:01,309 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f77d67d4ff5a489bb0ddfed9720a8caf_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f77d67d4ff5a489bb0ddfed9720a8caf_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:01,310 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/67454150efb9404f9340855e1772cc06, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:01,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/67454150efb9404f9340855e1772cc06 is 175, key is test_row_0/A:col10/1731494400873/Put/seqid=0 2024-11-13T10:40:01,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742268_1444 (size=48639) 2024-11-13T10:40:01,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494461317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:01,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494461317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T10:40:01,487 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:01,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-13T10:40:01,490 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:01,491 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:01,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T10:40:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T10:40:01,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494461627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494461630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:01,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T10:40:01,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:01,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:40:01,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:01,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:01,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:01,718 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/67454150efb9404f9340855e1772cc06 2024-11-13T10:40:01,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/6547af43298f426b836cbdd64e032fc9 is 50, key is test_row_0/B:col10/1731494400873/Put/seqid=0 2024-11-13T10:40:01,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742269_1445 (size=12301) 2024-11-13T10:40:01,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/6547af43298f426b836cbdd64e032fc9 2024-11-13T10:40:01,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/a7aecb5d5ecc455c8597b8fdb3a2a2aa is 50, key is test_row_0/C:col10/1731494400873/Put/seqid=0 2024-11-13T10:40:01,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742270_1446 (size=12301) 2024-11-13T10:40:01,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T10:40:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:40:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:01,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T10:40:01,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T10:40:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:40:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:01,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:02,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T10:40:02,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:02,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:40:02,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:02,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:02,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T10:40:02,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494462147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:02,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494462148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:02,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/a7aecb5d5ecc455c8597b8fdb3a2a2aa 2024-11-13T10:40:02,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/67454150efb9404f9340855e1772cc06 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/67454150efb9404f9340855e1772cc06 2024-11-13T10:40:02,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/67454150efb9404f9340855e1772cc06, entries=250, sequenceid=331, filesize=47.5 K 2024-11-13T10:40:02,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/6547af43298f426b836cbdd64e032fc9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6547af43298f426b836cbdd64e032fc9 2024-11-13T10:40:02,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6547af43298f426b836cbdd64e032fc9, entries=150, sequenceid=331, filesize=12.0 K 2024-11-13T10:40:02,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/a7aecb5d5ecc455c8597b8fdb3a2a2aa as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a7aecb5d5ecc455c8597b8fdb3a2a2aa 2024-11-13T10:40:02,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a7aecb5d5ecc455c8597b8fdb3a2a2aa, entries=150, sequenceid=331, filesize=12.0 K 2024-11-13T10:40:02,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 0b3f34f131c1584553eed8127d585c45 in 1323ms, sequenceid=331, compaction requested=true 2024-11-13T10:40:02,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:02,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:02,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:02,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:02,196 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:02,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:02,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:02,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:40:02,196 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:02,197 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120525 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:02,197 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/A is initiating minor compaction (all files) 2024-11-13T10:40:02,197 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/A in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:02,197 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9d024fcfc49c4792b11913e804cba350, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/c9c0cfa4e11d4b2189888cbc41a9e72f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/67454150efb9404f9340855e1772cc06] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=117.7 K 2024-11-13T10:40:02,197 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:02,197 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9d024fcfc49c4792b11913e804cba350, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/c9c0cfa4e11d4b2189888cbc41a9e72f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/67454150efb9404f9340855e1772cc06] 2024-11-13T10:40:02,197 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:02,197 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/B is initiating minor compaction (all files) 2024-11-13T10:40:02,197 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/B in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:02,197 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/38d1bcab325a422b964a75131e3b8f80, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/14a03196a1594c3cb566aa680b371dc9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6547af43298f426b836cbdd64e032fc9] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=36.7 K 2024-11-13T10:40:02,198 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d024fcfc49c4792b11913e804cba350, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731494399195 2024-11-13T10:40:02,198 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d1bcab325a422b964a75131e3b8f80, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731494399195 2024-11-13T10:40:02,198 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 14a03196a1594c3cb566aa680b371dc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731494400027 2024-11-13T10:40:02,198 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c0cfa4e11d4b2189888cbc41a9e72f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731494400011 2024-11-13T10:40:02,198 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6547af43298f426b836cbdd64e032fc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731494400207 2024-11-13T10:40:02,199 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67454150efb9404f9340855e1772cc06, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731494400203 2024-11-13T10:40:02,205 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:02,206 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:02,206 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/52e99d913e4045d1a637eea61c4a6fa9 is 50, key is test_row_0/B:col10/1731494400873/Put/seqid=0 2024-11-13T10:40:02,210 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113db60dd61b49c4e499755d875b9aa6e36_0b3f34f131c1584553eed8127d585c45 store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:02,212 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113db60dd61b49c4e499755d875b9aa6e36_0b3f34f131c1584553eed8127d585c45, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:02,212 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113db60dd61b49c4e499755d875b9aa6e36_0b3f34f131c1584553eed8127d585c45 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:02,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742271_1447 (size=13085) 2024-11-13T10:40:02,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742272_1448 (size=4469) 2024-11-13T10:40:02,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T10:40:02,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:02,266 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T10:40:02,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a5a01fdffc8944cb807c42e38a15042e_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494400952/Put/seqid=0 2024-11-13T10:40:02,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742273_1449 (size=12454) 2024-11-13T10:40:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T10:40:02,631 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/52e99d913e4045d1a637eea61c4a6fa9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/52e99d913e4045d1a637eea61c4a6fa9 2024-11-13T10:40:02,631 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#A#compaction#376 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:02,632 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f7eec37f3a0f418bb9249556a0e2fd10 is 175, key is test_row_0/A:col10/1731494400873/Put/seqid=0 2024-11-13T10:40:02,646 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/B of 0b3f34f131c1584553eed8127d585c45 into 52e99d913e4045d1a637eea61c4a6fa9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:02,646 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:02,646 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/B, priority=13, startTime=1731494402196; duration=0sec 2024-11-13T10:40:02,646 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:02,646 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:B 2024-11-13T10:40:02,646 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:02,647 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:02,648 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/C is initiating minor compaction (all files) 2024-11-13T10:40:02,648 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/C in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:02,648 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/fdc28452195f4800b8f579dce53f649d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/f3d9478196fe4a79becb117250b5f83f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a7aecb5d5ecc455c8597b8fdb3a2a2aa] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=36.7 K 2024-11-13T10:40:02,649 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc28452195f4800b8f579dce53f649d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731494399195 2024-11-13T10:40:02,650 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting f3d9478196fe4a79becb117250b5f83f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731494400027 2024-11-13T10:40:02,650 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting a7aecb5d5ecc455c8597b8fdb3a2a2aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731494400207 2024-11-13T10:40:02,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742274_1450 (size=32039) 2024-11-13T10:40:02,677 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/f7eec37f3a0f418bb9249556a0e2fd10 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f7eec37f3a0f418bb9249556a0e2fd10 2024-11-13T10:40:02,683 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#C#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:02,683 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/63c34f565f9f4c6b9f81a4d055fecd57 is 50, key is test_row_0/C:col10/1731494400873/Put/seqid=0 2024-11-13T10:40:02,688 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/A of 0b3f34f131c1584553eed8127d585c45 into f7eec37f3a0f418bb9249556a0e2fd10(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:02,688 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:02,688 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/A, priority=13, startTime=1731494402196; duration=0sec 2024-11-13T10:40:02,689 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:02,689 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:A 2024-11-13T10:40:02,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,693 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a5a01fdffc8944cb807c42e38a15042e_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a5a01fdffc8944cb807c42e38a15042e_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:02,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/781eade823044806b246f2f75b953c2a, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:02,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/781eade823044806b246f2f75b953c2a is 175, key is test_row_0/A:col10/1731494400952/Put/seqid=0 2024-11-13T10:40:02,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742275_1451 (size=13085) 2024-11-13T10:40:02,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742276_1452 (size=31255) 2024-11-13T10:40:02,745 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/781eade823044806b246f2f75b953c2a 2024-11-13T10:40:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/060b6f47eb0247f0a79a908e3479a7af is 50, key is test_row_0/B:col10/1731494400952/Put/seqid=0 2024-11-13T10:40:02,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742277_1453 (size=12301) 2024-11-13T10:40:02,831 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/060b6f47eb0247f0a79a908e3479a7af 2024-11-13T10:40:02,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ad80a76fff2648918444b40f751fb1b4 is 50, key is test_row_0/C:col10/1731494400952/Put/seqid=0 2024-11-13T10:40:02,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742278_1454 (size=12301) 2024-11-13T10:40:02,847 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ad80a76fff2648918444b40f751fb1b4 2024-11-13T10:40:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/781eade823044806b246f2f75b953c2a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/781eade823044806b246f2f75b953c2a 2024-11-13T10:40:02,867 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/781eade823044806b246f2f75b953c2a, entries=150, sequenceid=353, filesize=30.5 K 2024-11-13T10:40:02,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/060b6f47eb0247f0a79a908e3479a7af as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060b6f47eb0247f0a79a908e3479a7af 2024-11-13T10:40:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,872 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060b6f47eb0247f0a79a908e3479a7af, entries=150, sequenceid=353, filesize=12.0 K 2024-11-13T10:40:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/ad80a76fff2648918444b40f751fb1b4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ad80a76fff2648918444b40f751fb1b4 2024-11-13T10:40:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,878 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ad80a76fff2648918444b40f751fb1b4, entries=150, sequenceid=353, filesize=12.0 K 2024-11-13T10:40:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,879 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 0b3f34f131c1584553eed8127d585c45 in 614ms, sequenceid=353, compaction requested=false 2024-11-13T10:40:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-13T10:40:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-13T10:40:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-13T10:40:02,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3900 sec 2024-11-13T10:40:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.3970 sec 2024-11-13T10:40:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,122 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/63c34f565f9f4c6b9f81a4d055fecd57 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/63c34f565f9f4c6b9f81a4d055fecd57 2024-11-13T10:40:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,127 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/C of 0b3f34f131c1584553eed8127d585c45 into 63c34f565f9f4c6b9f81a4d055fecd57(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:03,127 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:03,127 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/C, priority=13, startTime=1731494402196; duration=0sec 2024-11-13T10:40:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,128 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:03,128 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:C 2024-11-13T10:40:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:03,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:03,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371d299d5a0ce469ebe579d477f7f9f27_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494403471/Put/seqid=0 2024-11-13T10:40:03,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742279_1455 (size=14994) 2024-11-13T10:40:03,591 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:03,594 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371d299d5a0ce469ebe579d477f7f9f27_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371d299d5a0ce469ebe579d477f7f9f27_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:03,595 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d80b5572b82a4b189fd8ddc020831912, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:03,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d80b5572b82a4b189fd8ddc020831912 is 175, key is test_row_0/A:col10/1731494403471/Put/seqid=0 2024-11-13T10:40:03,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742280_1456 (size=39949) 2024-11-13T10:40:03,632 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d80b5572b82a4b189fd8ddc020831912 2024-11-13T10:40:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T10:40:03,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:03,648 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:03,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/b986102d207a4a5292c9a8d7fa069ebd is 50, key is test_row_0/B:col10/1731494403471/Put/seqid=0 2024-11-13T10:40:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-13T10:40:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T10:40:03,653 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:03,654 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:03,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:03,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742281_1457 (size=12301) 2024-11-13T10:40:03,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/b986102d207a4a5292c9a8d7fa069ebd 2024-11-13T10:40:03,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/8cb7968235794e15be35278240266ed7 is 50, key is test_row_0/C:col10/1731494403471/Put/seqid=0 2024-11-13T10:40:03,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742282_1458 (size=12301) 2024-11-13T10:40:03,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494463726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:03,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494463727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T10:40:03,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T10:40:03,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:03,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:40:03,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:03,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:03,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:03,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494463848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:03,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494463848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:03,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T10:40:03,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:03,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:40:03,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:03,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:03,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T10:40:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:04,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494464061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:04,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494464072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:04,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T10:40:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:40:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:04,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:04,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:04,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/8cb7968235794e15be35278240266ed7 2024-11-13T10:40:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d80b5572b82a4b189fd8ddc020831912 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d80b5572b82a4b189fd8ddc020831912 2024-11-13T10:40:04,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d80b5572b82a4b189fd8ddc020831912, entries=200, sequenceid=367, filesize=39.0 K 2024-11-13T10:40:04,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/b986102d207a4a5292c9a8d7fa069ebd as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/b986102d207a4a5292c9a8d7fa069ebd 2024-11-13T10:40:04,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/b986102d207a4a5292c9a8d7fa069ebd, entries=150, sequenceid=367, filesize=12.0 K 2024-11-13T10:40:04,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/8cb7968235794e15be35278240266ed7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/8cb7968235794e15be35278240266ed7 2024-11-13T10:40:04,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/8cb7968235794e15be35278240266ed7, entries=150, sequenceid=367, filesize=12.0 K 2024-11-13T10:40:04,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 0b3f34f131c1584553eed8127d585c45 in 634ms, sequenceid=367, compaction requested=true 2024-11-13T10:40:04,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:04,170 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:04,170 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:04,172 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:04,172 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/A is initiating minor compaction (all files) 2024-11-13T10:40:04,172 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/A in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:04,172 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f7eec37f3a0f418bb9249556a0e2fd10, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/781eade823044806b246f2f75b953c2a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d80b5572b82a4b189fd8ddc020831912] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=100.8 K 2024-11-13T10:40:04,172 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:04,172 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f7eec37f3a0f418bb9249556a0e2fd10, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/781eade823044806b246f2f75b953c2a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d80b5572b82a4b189fd8ddc020831912] 2024-11-13T10:40:04,172 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:04,172 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/B is initiating minor compaction (all files) 2024-11-13T10:40:04,173 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/B in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:04,173 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/52e99d913e4045d1a637eea61c4a6fa9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060b6f47eb0247f0a79a908e3479a7af, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/b986102d207a4a5292c9a8d7fa069ebd] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=36.8 K 2024-11-13T10:40:04,173 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7eec37f3a0f418bb9249556a0e2fd10, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731494400207 2024-11-13T10:40:04,173 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 52e99d913e4045d1a637eea61c4a6fa9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731494400207 2024-11-13T10:40:04,173 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 060b6f47eb0247f0a79a908e3479a7af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731494400952 2024-11-13T10:40:04,173 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 781eade823044806b246f2f75b953c2a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731494400952 2024-11-13T10:40:04,174 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting b986102d207a4a5292c9a8d7fa069ebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731494403279 2024-11-13T10:40:04,174 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d80b5572b82a4b189fd8ddc020831912, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731494403279 2024-11-13T10:40:04,186 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:04,187 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/5f16c59641f84f0094a762ed48602be5 is 50, key is test_row_0/B:col10/1731494403471/Put/seqid=0 2024-11-13T10:40:04,194 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:04,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742283_1459 (size=13187) 2024-11-13T10:40:04,227 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/5f16c59641f84f0094a762ed48602be5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/5f16c59641f84f0094a762ed48602be5 2024-11-13T10:40:04,233 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113cc1508ac8684496ab107ee23240d5549_0b3f34f131c1584553eed8127d585c45 store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:04,234 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/B of 0b3f34f131c1584553eed8127d585c45 into 5f16c59641f84f0094a762ed48602be5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:04,234 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:04,234 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/B, priority=13, startTime=1731494404170; duration=0sec 2024-11-13T10:40:04,235 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:04,235 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:B 2024-11-13T10:40:04,235 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:04,235 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113cc1508ac8684496ab107ee23240d5549_0b3f34f131c1584553eed8127d585c45, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:04,235 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cc1508ac8684496ab107ee23240d5549_0b3f34f131c1584553eed8127d585c45 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:04,237 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:04,237 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): 0b3f34f131c1584553eed8127d585c45/C is initiating minor compaction (all files) 2024-11-13T10:40:04,237 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b3f34f131c1584553eed8127d585c45/C in TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:04,237 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/63c34f565f9f4c6b9f81a4d055fecd57, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ad80a76fff2648918444b40f751fb1b4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/8cb7968235794e15be35278240266ed7] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp, totalSize=36.8 K 2024-11-13T10:40:04,237 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 63c34f565f9f4c6b9f81a4d055fecd57, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731494400207 2024-11-13T10:40:04,238 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ad80a76fff2648918444b40f751fb1b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731494400952 2024-11-13T10:40:04,238 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb7968235794e15be35278240266ed7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731494403279 2024-11-13T10:40:04,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742284_1460 (size=4469) 2024-11-13T10:40:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T10:40:04,286 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:04,287 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/728b9eb81c15436ca9e96bea851bd624 is 50, key is test_row_0/C:col10/1731494403471/Put/seqid=0 2024-11-13T10:40:04,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T10:40:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:04,290 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:40:04,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a409cfa8673f45cbb1aab04a180e8d7f_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494403703/Put/seqid=0 2024-11-13T10:40:04,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742285_1461 (size=13187) 2024-11-13T10:40:04,334 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/728b9eb81c15436ca9e96bea851bd624 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/728b9eb81c15436ca9e96bea851bd624 2024-11-13T10:40:04,345 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/C of 0b3f34f131c1584553eed8127d585c45 into 728b9eb81c15436ca9e96bea851bd624(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:04,345 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:04,345 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/C, priority=13, startTime=1731494404170; duration=0sec 2024-11-13T10:40:04,345 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:04,345 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:C 2024-11-13T10:40:04,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742286_1462 (size=12454) 2024-11-13T10:40:04,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:04,358 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a409cfa8673f45cbb1aab04a180e8d7f_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a409cfa8673f45cbb1aab04a180e8d7f_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/313c83b374e5484d811e300ff5470488, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/313c83b374e5484d811e300ff5470488 is 175, key is test_row_0/A:col10/1731494403703/Put/seqid=0 2024-11-13T10:40:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:04,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. as already flushing 2024-11-13T10:40:04,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742287_1463 (size=31255) 2024-11-13T10:40:04,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494464416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:04,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494464425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:04,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494464527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:04,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494464547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:04,671 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b3f34f131c1584553eed8127d585c45#A#compaction#385 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:04,672 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/8259ee039f1140a5b8667eca6cf76d77 is 175, key is test_row_0/A:col10/1731494403471/Put/seqid=0 2024-11-13T10:40:04,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742288_1464 (size=32141) 2024-11-13T10:40:04,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494464748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:04,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:04,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494464756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:04,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T10:40:04,793 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/313c83b374e5484d811e300ff5470488 2024-11-13T10:40:04,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/ded5b0f4c2004eeb86bd58cc08193ba9 is 50, key is test_row_0/B:col10/1731494403703/Put/seqid=0 2024-11-13T10:40:04,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742289_1465 (size=12301) 2024-11-13T10:40:05,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494465057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:05,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:05,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494465076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:05,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:05,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:05,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:05,108 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/8259ee039f1140a5b8667eca6cf76d77 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/8259ee039f1140a5b8667eca6cf76d77 2024-11-13T10:40:05,114 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b3f34f131c1584553eed8127d585c45/A of 0b3f34f131c1584553eed8127d585c45 into 8259ee039f1140a5b8667eca6cf76d77(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:05,114 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:05,114 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., storeName=0b3f34f131c1584553eed8127d585c45/A, priority=13, startTime=1731494404169; duration=0sec 2024-11-13T10:40:05,114 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:05,114 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:A 2024-11-13T10:40:05,213 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/ded5b0f4c2004eeb86bd58cc08193ba9 2024-11-13T10:40:05,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/54253d637bcd477c9f749f264f33818d is 50, key is test_row_0/C:col10/1731494403703/Put/seqid=0 2024-11-13T10:40:05,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742290_1466 (size=12301) 2024-11-13T10:40:05,532 INFO [Thread-1617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1617. 2024-11-13T10:40:05,532 DEBUG [Thread-1617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:05,532 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:05,532 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:05,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:05,533 INFO [Thread-1613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1613. 2024-11-13T10:40:05,533 DEBUG [Thread-1613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:05,533 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:05,533 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:05,533 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:05,536 INFO [Thread-1619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1619. 2024-11-13T10:40:05,536 DEBUG [Thread-1619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:05,536 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:05,537 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:05,537 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:05,541 INFO [Thread-1621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1621. 2024-11-13T10:40:05,541 INFO [Thread-1615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1615. 2024-11-13T10:40:05,541 DEBUG [Thread-1621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:05,541 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:05,541 DEBUG [Thread-1615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:05,541 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:05,541 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:05,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:05,542 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:05,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:05,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57488 deadline: 1731494465577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:05,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57452 deadline: 1731494465597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:05,636 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/54253d637bcd477c9f749f264f33818d 2024-11-13T10:40:05,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/313c83b374e5484d811e300ff5470488 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/313c83b374e5484d811e300ff5470488 2024-11-13T10:40:05,643 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/313c83b374e5484d811e300ff5470488, entries=150, sequenceid=394, filesize=30.5 K 2024-11-13T10:40:05,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/ded5b0f4c2004eeb86bd58cc08193ba9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/ded5b0f4c2004eeb86bd58cc08193ba9 2024-11-13T10:40:05,647 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/ded5b0f4c2004eeb86bd58cc08193ba9, entries=150, sequenceid=394, filesize=12.0 K 2024-11-13T10:40:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/54253d637bcd477c9f749f264f33818d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/54253d637bcd477c9f749f264f33818d 2024-11-13T10:40:05,651 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/54253d637bcd477c9f749f264f33818d, entries=150, sequenceid=394, filesize=12.0 K 2024-11-13T10:40:05,652 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0b3f34f131c1584553eed8127d585c45 in 1362ms, sequenceid=394, compaction requested=false 2024-11-13T10:40:05,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:05,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:05,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-13T10:40:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-13T10:40:05,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-13T10:40:05,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0000 sec 2024-11-13T10:40:05,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.0080 sec 2024-11-13T10:40:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T10:40:05,797 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:06,598 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:06,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:06,598 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-11-13T10:40:06,598 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:06,598 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:06,598 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:06,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:06,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111328513a16d75849e6819a5dc8afad81cc_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494404414/Put/seqid=0 2024-11-13T10:40:06,608 INFO [Thread-1608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1608. 2024-11-13T10:40:06,608 DEBUG [Thread-1608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:06,608 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:06,609 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:06,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:06,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742291_1467 (size=12454) 2024-11-13T10:40:07,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:07,013 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111328513a16d75849e6819a5dc8afad81cc_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111328513a16d75849e6819a5dc8afad81cc_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:07,014 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d70e7abe0911459da27a6e5f56b96e77, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:07,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d70e7abe0911459da27a6e5f56b96e77 is 175, key is test_row_0/A:col10/1731494404414/Put/seqid=0 2024-11-13T10:40:07,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742292_1468 (size=31255) 2024-11-13T10:40:07,418 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=407, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d70e7abe0911459da27a6e5f56b96e77 2024-11-13T10:40:07,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/7e846ab581134fae9c971a3fb671d199 is 50, key is test_row_0/B:col10/1731494404414/Put/seqid=0 2024-11-13T10:40:07,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742293_1469 (size=12301) 2024-11-13T10:40:07,659 INFO [Thread-1606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1606. 2024-11-13T10:40:07,659 DEBUG [Thread-1606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:07,659 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:07,659 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:07,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:07,668 INFO [Thread-1602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1602. 2024-11-13T10:40:07,668 DEBUG [Thread-1602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:07,669 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:07,669 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:07,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:07,698 INFO [Thread-1604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1604. 2024-11-13T10:40:07,698 DEBUG [Thread-1604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:07,698 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:07,698 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:07,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:07,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 140 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1054 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3152 rows 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1058 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3173 rows 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1058 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3167 rows 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1054 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3162 rows 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1057 2024-11-13T10:40:07,699 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3164 rows 2024-11-13T10:40:07,699 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T10:40:07,699 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T10:40:07,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:07,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:07,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:07,700 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T10:40:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T10:40:07,703 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494407703"}]},"ts":"1731494407703"} 2024-11-13T10:40:07,704 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T10:40:07,704 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T10:40:07,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T10:40:07,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, UNASSIGN}] 2024-11-13T10:40:07,707 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, UNASSIGN 2024-11-13T10:40:07,707 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=0b3f34f131c1584553eed8127d585c45, regionState=CLOSING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:40:07,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, UNASSIGN because future has completed 2024-11-13T10:40:07,709 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T10:40:07,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:40:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T10:40:07,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/7e846ab581134fae9c971a3fb671d199 2024-11-13T10:40:07,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/2c0aabb274334dec94de8324689b61b1 is 50, key is test_row_0/C:col10/1731494404414/Put/seqid=0 2024-11-13T10:40:07,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742294_1470 (size=12301) 2024-11-13T10:40:07,861 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:07,861 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T10:40:07,861 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 0b3f34f131c1584553eed8127d585c45, disabling compactions & flushes 2024-11-13T10:40:07,861 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:08,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T10:40:08,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/2c0aabb274334dec94de8324689b61b1 2024-11-13T10:40:08,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/d70e7abe0911459da27a6e5f56b96e77 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d70e7abe0911459da27a6e5f56b96e77 2024-11-13T10:40:08,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d70e7abe0911459da27a6e5f56b96e77, entries=150, sequenceid=407, filesize=30.5 K 2024-11-13T10:40:08,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/7e846ab581134fae9c971a3fb671d199 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/7e846ab581134fae9c971a3fb671d199 2024-11-13T10:40:08,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/7e846ab581134fae9c971a3fb671d199, entries=150, sequenceid=407, filesize=12.0 K 2024-11-13T10:40:08,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/2c0aabb274334dec94de8324689b61b1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/2c0aabb274334dec94de8324689b61b1 2024-11-13T10:40:08,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/2c0aabb274334dec94de8324689b61b1, entries=150, sequenceid=407, filesize=12.0 K 2024-11-13T10:40:08,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=26.84 KB/27480 for 0b3f34f131c1584553eed8127d585c45 in 1657ms, sequenceid=407, compaction requested=true 2024-11-13T10:40:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b3f34f131c1584553eed8127d585c45: 2024-11-13T10:40:08,255 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:08,255 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:08,255 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. after waiting 0 ms 2024-11-13T10:40:08,255 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. because compaction request was cancelled 2024-11-13T10:40:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:08,255 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:08,255 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:A 2024-11-13T10:40:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:B, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:08,255 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. because compaction request was cancelled 2024-11-13T10:40:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b3f34f131c1584553eed8127d585c45:C, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:08,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:08,255 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. because compaction request was cancelled 2024-11-13T10:40:08,255 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 0b3f34f131c1584553eed8127d585c45 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-13T10:40:08,255 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:B 2024-11-13T10:40:08,255 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b3f34f131c1584553eed8127d585c45:C 2024-11-13T10:40:08,261 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bfe0463848f042caa537c5a2563e867c_0b3f34f131c1584553eed8127d585c45 is 50, key is test_row_0/A:col10/1731494407657/Put/seqid=0 2024-11-13T10:40:08,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742295_1471 (size=12454) 2024-11-13T10:40:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T10:40:08,665 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:08,668 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bfe0463848f042caa537c5a2563e867c_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bfe0463848f042caa537c5a2563e867c_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:08,669 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/1bfe037e36504a4c88c5181cb115d512, store: [table=TestAcidGuarantees family=A region=0b3f34f131c1584553eed8127d585c45] 2024-11-13T10:40:08,670 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/1bfe037e36504a4c88c5181cb115d512 is 175, key is test_row_0/A:col10/1731494407657/Put/seqid=0 2024-11-13T10:40:08,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742296_1472 (size=31255) 2024-11-13T10:40:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T10:40:09,074 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/1bfe037e36504a4c88c5181cb115d512 2024-11-13T10:40:09,080 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/14e848c8a224464a809be8bc66bf4d45 is 50, key is test_row_0/B:col10/1731494407657/Put/seqid=0 2024-11-13T10:40:09,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742297_1473 (size=12301) 2024-11-13T10:40:09,485 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/14e848c8a224464a809be8bc66bf4d45 2024-11-13T10:40:09,490 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/d26488e3892a4ae7b41f9dfaf262c751 is 50, key is test_row_0/C:col10/1731494407657/Put/seqid=0 2024-11-13T10:40:09,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742298_1474 (size=12301) 2024-11-13T10:40:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T10:40:09,894 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/d26488e3892a4ae7b41f9dfaf262c751 2024-11-13T10:40:09,898 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/A/1bfe037e36504a4c88c5181cb115d512 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/1bfe037e36504a4c88c5181cb115d512 2024-11-13T10:40:09,902 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/1bfe037e36504a4c88c5181cb115d512, entries=150, sequenceid=414, filesize=30.5 K 2024-11-13T10:40:09,903 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/B/14e848c8a224464a809be8bc66bf4d45 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/14e848c8a224464a809be8bc66bf4d45 2024-11-13T10:40:09,907 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/14e848c8a224464a809be8bc66bf4d45, entries=150, sequenceid=414, filesize=12.0 K 2024-11-13T10:40:09,907 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/.tmp/C/d26488e3892a4ae7b41f9dfaf262c751 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/d26488e3892a4ae7b41f9dfaf262c751 2024-11-13T10:40:09,910 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/d26488e3892a4ae7b41f9dfaf262c751, entries=150, sequenceid=414, filesize=12.0 K 2024-11-13T10:40:09,911 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0b3f34f131c1584553eed8127d585c45 in 1656ms, sequenceid=414, compaction requested=true 2024-11-13T10:40:09,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/61b5033932e446429a89e61a7a6cb423, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f6393b1e29cc444c99a7b57b10170210, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b1606424455b4346bffe09897a5a6a4a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/e5bdf5de0c8e407c89b7c198a1cda307, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/287977efe18f48c3a83f6949c3d0f107, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/20726465897f4894834cdf8868e17da4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f5790f6e7b2c4f2697df69f0e919634c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7e7d2ddb73fe4325832950fc7706707f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/8a81a5496aaa4ef8bcf17335226b213d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b3b989a77fc942d9af28aa136cf66f0d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9714ae9fbd214ed9b4d14cfe6a8ead46, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/210d7ddb516c48ce9a40d521b8f1fd25, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/af16d0f7eab54aa28c68007cf912de92, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/a8ffc71e0d97477180b9204a56b0f5f5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d217c9e2bd274c978d52f8929a28bf0f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d487cc65fdad4c21a33a1bcf824ff525, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f2652d16e63849d48cf2235eddee15de, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/55a8751b58784a0682182b287b1af9f7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/ad3bb10044ab4680bd3ba07cb088551f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7b0315e25af842269cd4d7de09680f06, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9d024fcfc49c4792b11913e804cba350, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/c9c0cfa4e11d4b2189888cbc41a9e72f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/67454150efb9404f9340855e1772cc06, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f7eec37f3a0f418bb9249556a0e2fd10, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/781eade823044806b246f2f75b953c2a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d80b5572b82a4b189fd8ddc020831912] to archive 2024-11-13T10:40:09,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:40:09,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/61b5033932e446429a89e61a7a6cb423 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/61b5033932e446429a89e61a7a6cb423 2024-11-13T10:40:09,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f6393b1e29cc444c99a7b57b10170210 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f6393b1e29cc444c99a7b57b10170210 2024-11-13T10:40:09,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b1606424455b4346bffe09897a5a6a4a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b1606424455b4346bffe09897a5a6a4a 2024-11-13T10:40:09,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/e5bdf5de0c8e407c89b7c198a1cda307 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/e5bdf5de0c8e407c89b7c198a1cda307 2024-11-13T10:40:09,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/287977efe18f48c3a83f6949c3d0f107 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/287977efe18f48c3a83f6949c3d0f107 2024-11-13T10:40:09,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/20726465897f4894834cdf8868e17da4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/20726465897f4894834cdf8868e17da4 2024-11-13T10:40:09,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f5790f6e7b2c4f2697df69f0e919634c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f5790f6e7b2c4f2697df69f0e919634c 2024-11-13T10:40:09,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7e7d2ddb73fe4325832950fc7706707f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7e7d2ddb73fe4325832950fc7706707f 2024-11-13T10:40:09,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/8a81a5496aaa4ef8bcf17335226b213d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/8a81a5496aaa4ef8bcf17335226b213d 2024-11-13T10:40:09,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b3b989a77fc942d9af28aa136cf66f0d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/b3b989a77fc942d9af28aa136cf66f0d 2024-11-13T10:40:09,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9714ae9fbd214ed9b4d14cfe6a8ead46 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9714ae9fbd214ed9b4d14cfe6a8ead46 2024-11-13T10:40:09,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/210d7ddb516c48ce9a40d521b8f1fd25 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/210d7ddb516c48ce9a40d521b8f1fd25 2024-11-13T10:40:09,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/af16d0f7eab54aa28c68007cf912de92 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/af16d0f7eab54aa28c68007cf912de92 2024-11-13T10:40:09,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/a8ffc71e0d97477180b9204a56b0f5f5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/a8ffc71e0d97477180b9204a56b0f5f5 2024-11-13T10:40:09,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d217c9e2bd274c978d52f8929a28bf0f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d217c9e2bd274c978d52f8929a28bf0f 2024-11-13T10:40:09,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d487cc65fdad4c21a33a1bcf824ff525 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d487cc65fdad4c21a33a1bcf824ff525 2024-11-13T10:40:09,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f2652d16e63849d48cf2235eddee15de to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f2652d16e63849d48cf2235eddee15de 2024-11-13T10:40:09,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/55a8751b58784a0682182b287b1af9f7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/55a8751b58784a0682182b287b1af9f7 2024-11-13T10:40:09,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/ad3bb10044ab4680bd3ba07cb088551f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/ad3bb10044ab4680bd3ba07cb088551f 2024-11-13T10:40:09,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7b0315e25af842269cd4d7de09680f06 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/7b0315e25af842269cd4d7de09680f06 2024-11-13T10:40:09,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9d024fcfc49c4792b11913e804cba350 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/9d024fcfc49c4792b11913e804cba350 2024-11-13T10:40:09,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/c9c0cfa4e11d4b2189888cbc41a9e72f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/c9c0cfa4e11d4b2189888cbc41a9e72f 2024-11-13T10:40:09,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/67454150efb9404f9340855e1772cc06 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/67454150efb9404f9340855e1772cc06 2024-11-13T10:40:09,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f7eec37f3a0f418bb9249556a0e2fd10 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/f7eec37f3a0f418bb9249556a0e2fd10 2024-11-13T10:40:09,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/781eade823044806b246f2f75b953c2a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/781eade823044806b246f2f75b953c2a 2024-11-13T10:40:09,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d80b5572b82a4b189fd8ddc020831912 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d80b5572b82a4b189fd8ddc020831912 2024-11-13T10:40:09,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/7a14686e2d7b41e79506b9829ed73a11, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e622d20580884c599210e123e8002aec, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/bcdfbbff63ff42808daae1f3720308f6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/391ba4692c63461b819b10cb0061e1f8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/ef00af45f8fb4640b76d1b138e5b6f3a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/23cb9717b85b49b7bef321f117a90634, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/c8e028a690a94adb92eede45876df54d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/19933c8ef2564d309414bb822d9fb1d1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/73ed2270cf4144f7bc8f516d134565a8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6c39331105e54810a8943e5587084547, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e3ba396bfecb4affb1efe6c76b65ba20, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/69a4567296954bc8bc9ee23e9cb342ab, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060d1644e5364b6d8ffee42c7a81f19d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/718bfab91dbb44a7ba7e4725e4c21930, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/4d40c30a0dae478c950fd25fea64402d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/04fb3eb3668a461c8bec35e6bba5d4bd, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/cb046275ae7e4bd8ba9e4d4f6d540993, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/061f4df175284a49b5f4f302adcbe623, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e56ca6d0395540f2aa93f20e81be520a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/38d1bcab325a422b964a75131e3b8f80, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/47f0de69b50f4cf7a2ed00687385684d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/14a03196a1594c3cb566aa680b371dc9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/52e99d913e4045d1a637eea61c4a6fa9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6547af43298f426b836cbdd64e032fc9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060b6f47eb0247f0a79a908e3479a7af, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/b986102d207a4a5292c9a8d7fa069ebd] to archive 2024-11-13T10:40:09,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:40:09,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/7a14686e2d7b41e79506b9829ed73a11 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/7a14686e2d7b41e79506b9829ed73a11 2024-11-13T10:40:09,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e622d20580884c599210e123e8002aec to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e622d20580884c599210e123e8002aec 2024-11-13T10:40:09,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/bcdfbbff63ff42808daae1f3720308f6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/bcdfbbff63ff42808daae1f3720308f6 2024-11-13T10:40:09,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/391ba4692c63461b819b10cb0061e1f8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/391ba4692c63461b819b10cb0061e1f8 2024-11-13T10:40:09,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/ef00af45f8fb4640b76d1b138e5b6f3a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/ef00af45f8fb4640b76d1b138e5b6f3a 2024-11-13T10:40:09,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/23cb9717b85b49b7bef321f117a90634 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/23cb9717b85b49b7bef321f117a90634 2024-11-13T10:40:09,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/c8e028a690a94adb92eede45876df54d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/c8e028a690a94adb92eede45876df54d 2024-11-13T10:40:09,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/19933c8ef2564d309414bb822d9fb1d1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/19933c8ef2564d309414bb822d9fb1d1 2024-11-13T10:40:09,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/73ed2270cf4144f7bc8f516d134565a8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/73ed2270cf4144f7bc8f516d134565a8 2024-11-13T10:40:09,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6c39331105e54810a8943e5587084547 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6c39331105e54810a8943e5587084547 2024-11-13T10:40:09,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e3ba396bfecb4affb1efe6c76b65ba20 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e3ba396bfecb4affb1efe6c76b65ba20 2024-11-13T10:40:09,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/69a4567296954bc8bc9ee23e9cb342ab to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/69a4567296954bc8bc9ee23e9cb342ab 2024-11-13T10:40:09,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060d1644e5364b6d8ffee42c7a81f19d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060d1644e5364b6d8ffee42c7a81f19d 2024-11-13T10:40:09,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/718bfab91dbb44a7ba7e4725e4c21930 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/718bfab91dbb44a7ba7e4725e4c21930 2024-11-13T10:40:09,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/4d40c30a0dae478c950fd25fea64402d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/4d40c30a0dae478c950fd25fea64402d 2024-11-13T10:40:09,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/04fb3eb3668a461c8bec35e6bba5d4bd to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/04fb3eb3668a461c8bec35e6bba5d4bd 2024-11-13T10:40:09,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/cb046275ae7e4bd8ba9e4d4f6d540993 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/cb046275ae7e4bd8ba9e4d4f6d540993 2024-11-13T10:40:09,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/061f4df175284a49b5f4f302adcbe623 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/061f4df175284a49b5f4f302adcbe623 2024-11-13T10:40:09,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e56ca6d0395540f2aa93f20e81be520a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/e56ca6d0395540f2aa93f20e81be520a 2024-11-13T10:40:09,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/38d1bcab325a422b964a75131e3b8f80 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/38d1bcab325a422b964a75131e3b8f80 2024-11-13T10:40:09,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/47f0de69b50f4cf7a2ed00687385684d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/47f0de69b50f4cf7a2ed00687385684d 2024-11-13T10:40:09,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/14a03196a1594c3cb566aa680b371dc9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/14a03196a1594c3cb566aa680b371dc9 2024-11-13T10:40:09,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/52e99d913e4045d1a637eea61c4a6fa9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/52e99d913e4045d1a637eea61c4a6fa9 2024-11-13T10:40:09,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6547af43298f426b836cbdd64e032fc9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/6547af43298f426b836cbdd64e032fc9 2024-11-13T10:40:09,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060b6f47eb0247f0a79a908e3479a7af to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/060b6f47eb0247f0a79a908e3479a7af 2024-11-13T10:40:09,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/b986102d207a4a5292c9a8d7fa069ebd to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/b986102d207a4a5292c9a8d7fa069ebd 2024-11-13T10:40:09,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ee48d2b779344cedb628833bed1147a5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/94874eeb7e18400a8fc039b27cd7607e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/2b8fb9f17a524d8d90aa67db31ff2fd4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ff7dd41b89604d3dbf5359c7f0e1e33a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/7737e125c6634f1584f6697aee4033ea, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/5394d1bcd7a64430955ff4991d69e0c1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/be829710bdab4f95822f6e30dc5e882a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/eb1a44749354420c85f3159df6281254, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/378ef630fc0547b090ff8a065cea7521, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ce63c286b3d440518bdf2ffa2ce14f23, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/9ab8368a62144265bcef6432894e2a63, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/0d544ba1ae684f6bbfaf50033221de06, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/009d70f8fe4b4c3494c4d50937b9bcae, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6b187854e32c4ce890879ebcddfcf2de, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/d6bc9ec6455a496588f430f9828cbd5c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/747d90bce72448939cebe7361e861f57, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a3d73c7dea1d4fe480e548d350c2746f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/c164655c97dc46cf995dd57d5b8f9e7b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/55b674a271344bb2b21e2d543d0daf1a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/fdc28452195f4800b8f579dce53f649d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6e892582b4784b09be7419a0fd60c739, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/f3d9478196fe4a79becb117250b5f83f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/63c34f565f9f4c6b9f81a4d055fecd57, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a7aecb5d5ecc455c8597b8fdb3a2a2aa, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ad80a76fff2648918444b40f751fb1b4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/8cb7968235794e15be35278240266ed7] to archive 2024-11-13T10:40:09,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:40:09,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ee48d2b779344cedb628833bed1147a5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ee48d2b779344cedb628833bed1147a5 2024-11-13T10:40:09,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/94874eeb7e18400a8fc039b27cd7607e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/94874eeb7e18400a8fc039b27cd7607e 2024-11-13T10:40:09,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/2b8fb9f17a524d8d90aa67db31ff2fd4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/2b8fb9f17a524d8d90aa67db31ff2fd4 2024-11-13T10:40:09,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ff7dd41b89604d3dbf5359c7f0e1e33a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ff7dd41b89604d3dbf5359c7f0e1e33a 2024-11-13T10:40:09,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/7737e125c6634f1584f6697aee4033ea to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/7737e125c6634f1584f6697aee4033ea 2024-11-13T10:40:09,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/5394d1bcd7a64430955ff4991d69e0c1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/5394d1bcd7a64430955ff4991d69e0c1 2024-11-13T10:40:09,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/be829710bdab4f95822f6e30dc5e882a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/be829710bdab4f95822f6e30dc5e882a 2024-11-13T10:40:09,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/eb1a44749354420c85f3159df6281254 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/eb1a44749354420c85f3159df6281254 2024-11-13T10:40:09,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/378ef630fc0547b090ff8a065cea7521 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/378ef630fc0547b090ff8a065cea7521 2024-11-13T10:40:09,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ce63c286b3d440518bdf2ffa2ce14f23 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ce63c286b3d440518bdf2ffa2ce14f23 2024-11-13T10:40:09,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/9ab8368a62144265bcef6432894e2a63 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/9ab8368a62144265bcef6432894e2a63 2024-11-13T10:40:09,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/0d544ba1ae684f6bbfaf50033221de06 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/0d544ba1ae684f6bbfaf50033221de06 2024-11-13T10:40:09,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/009d70f8fe4b4c3494c4d50937b9bcae to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/009d70f8fe4b4c3494c4d50937b9bcae 2024-11-13T10:40:09,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6b187854e32c4ce890879ebcddfcf2de to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6b187854e32c4ce890879ebcddfcf2de 2024-11-13T10:40:09,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/d6bc9ec6455a496588f430f9828cbd5c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/d6bc9ec6455a496588f430f9828cbd5c 2024-11-13T10:40:09,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/747d90bce72448939cebe7361e861f57 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/747d90bce72448939cebe7361e861f57 2024-11-13T10:40:09,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a3d73c7dea1d4fe480e548d350c2746f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a3d73c7dea1d4fe480e548d350c2746f 2024-11-13T10:40:09,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/c164655c97dc46cf995dd57d5b8f9e7b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/c164655c97dc46cf995dd57d5b8f9e7b 2024-11-13T10:40:09,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/55b674a271344bb2b21e2d543d0daf1a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/55b674a271344bb2b21e2d543d0daf1a 2024-11-13T10:40:09,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/fdc28452195f4800b8f579dce53f649d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/fdc28452195f4800b8f579dce53f649d 2024-11-13T10:40:09,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6e892582b4784b09be7419a0fd60c739 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/6e892582b4784b09be7419a0fd60c739 2024-11-13T10:40:09,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/f3d9478196fe4a79becb117250b5f83f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/f3d9478196fe4a79becb117250b5f83f 2024-11-13T10:40:09,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/63c34f565f9f4c6b9f81a4d055fecd57 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/63c34f565f9f4c6b9f81a4d055fecd57 2024-11-13T10:40:09,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a7aecb5d5ecc455c8597b8fdb3a2a2aa to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/a7aecb5d5ecc455c8597b8fdb3a2a2aa 2024-11-13T10:40:09,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ad80a76fff2648918444b40f751fb1b4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/ad80a76fff2648918444b40f751fb1b4 2024-11-13T10:40:09,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/8cb7968235794e15be35278240266ed7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/8cb7968235794e15be35278240266ed7 2024-11-13T10:40:09,996 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/recovered.edits/417.seqid, newMaxSeqId=417, maxSeqId=4 2024-11-13T10:40:09,997 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. 2024-11-13T10:40:09,997 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 0b3f34f131c1584553eed8127d585c45: Waiting for close lock at 1731494407861Running coprocessor pre-close hooks at 1731494407861Disabling compacts and flushes for region at 1731494407861Disabling writes for close at 1731494408255 (+394 ms)Obtaining lock to block concurrent updates at 1731494408255Preparing flush snapshotting stores in 0b3f34f131c1584553eed8127d585c45 at 1731494408255Finished memstore snapshotting TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731494408256 (+1 ms)Flushing stores of TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45. at 1731494408256Flushing 0b3f34f131c1584553eed8127d585c45/A: creating writer at 1731494408256Flushing mob file 0b3f34f131c1584553eed8127d585c45/A: appending metadata at 1731494408260 (+4 ms)Flushing mob file 0b3f34f131c1584553eed8127d585c45/A: closing flushed file at 1731494408260Flushing 0b3f34f131c1584553eed8127d585c45/A: appending metadata at 1731494408669 (+409 ms)Flushing 0b3f34f131c1584553eed8127d585c45/A: closing flushed file at 1731494408670 (+1 ms)Flushing 0b3f34f131c1584553eed8127d585c45/B: creating writer at 1731494409078 (+408 ms)Flushing 0b3f34f131c1584553eed8127d585c45/B: appending metadata at 1731494409080 (+2 ms)Flushing 0b3f34f131c1584553eed8127d585c45/B: closing flushed file at 1731494409080Flushing 0b3f34f131c1584553eed8127d585c45/C: creating writer at 1731494409488 (+408 ms)Flushing 0b3f34f131c1584553eed8127d585c45/C: appending metadata at 1731494409490 (+2 ms)Flushing 0b3f34f131c1584553eed8127d585c45/C: closing flushed file at 1731494409490Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66835ee7: reopening flushed file at 1731494409898 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1403d3f2: reopening flushed file at 1731494409902 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3997f19e: reopening flushed file at 1731494409907 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0b3f34f131c1584553eed8127d585c45 in 1656ms, sequenceid=414, compaction requested=true at 1731494409911 (+4 ms)Writing region close event to WAL at 1731494409994 (+83 ms)Running coprocessor post-close hooks at 1731494409997 (+3 ms)Closed at 1731494409997 2024-11-13T10:40:09,999 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:09,999 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=0b3f34f131c1584553eed8127d585c45, regionState=CLOSED 2024-11-13T10:40:10,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:40:10,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-13T10:40:10,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 0b3f34f131c1584553eed8127d585c45, server=770665a7984d,39101,1731494292956 in 2.2920 sec 2024-11-13T10:40:10,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-13T10:40:10,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b3f34f131c1584553eed8127d585c45, UNASSIGN in 2.2960 sec 2024-11-13T10:40:10,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-13T10:40:10,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3000 sec 2024-11-13T10:40:10,007 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494410007"}]},"ts":"1731494410007"} 2024-11-13T10:40:10,009 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T10:40:10,009 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T10:40:10,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3090 sec 2024-11-13T10:40:11,030 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T10:40:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T10:40:11,868 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:11,868 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T10:40:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:11,870 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T10:40:11,871 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:11,873 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,875 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/recovered.edits] 2024-11-13T10:40:11,878 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/1bfe037e36504a4c88c5181cb115d512 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/1bfe037e36504a4c88c5181cb115d512 2024-11-13T10:40:11,879 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/313c83b374e5484d811e300ff5470488 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/313c83b374e5484d811e300ff5470488 2024-11-13T10:40:11,881 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/8259ee039f1140a5b8667eca6cf76d77 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/8259ee039f1140a5b8667eca6cf76d77 2024-11-13T10:40:11,888 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d70e7abe0911459da27a6e5f56b96e77 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/A/d70e7abe0911459da27a6e5f56b96e77 2024-11-13T10:40:11,902 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/14e848c8a224464a809be8bc66bf4d45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/14e848c8a224464a809be8bc66bf4d45 2024-11-13T10:40:11,907 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/5f16c59641f84f0094a762ed48602be5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/5f16c59641f84f0094a762ed48602be5 2024-11-13T10:40:11,909 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/7e846ab581134fae9c971a3fb671d199 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/7e846ab581134fae9c971a3fb671d199 2024-11-13T10:40:11,910 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/ded5b0f4c2004eeb86bd58cc08193ba9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/B/ded5b0f4c2004eeb86bd58cc08193ba9 2024-11-13T10:40:11,913 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/2c0aabb274334dec94de8324689b61b1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/2c0aabb274334dec94de8324689b61b1 2024-11-13T10:40:11,914 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/54253d637bcd477c9f749f264f33818d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/54253d637bcd477c9f749f264f33818d 2024-11-13T10:40:11,921 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/728b9eb81c15436ca9e96bea851bd624 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/728b9eb81c15436ca9e96bea851bd624 2024-11-13T10:40:11,923 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/d26488e3892a4ae7b41f9dfaf262c751 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/C/d26488e3892a4ae7b41f9dfaf262c751 2024-11-13T10:40:11,926 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/recovered.edits/417.seqid to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45/recovered.edits/417.seqid 2024-11-13T10:40:11,926 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,926 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T10:40:11,927 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T10:40:11,928 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T10:40:11,930 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111305b060e6dc0646d391e1aef405303e43_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111305b060e6dc0646d391e1aef405303e43_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,932 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111309c45c2c1ccc4e22bb2caf6d87c89d3e_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111309c45c2c1ccc4e22bb2caf6d87c89d3e_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,933 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130d098e0f65ee4d53add6f9d3be57b3b9_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130d098e0f65ee4d53add6f9d3be57b3b9_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,934 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111312fab2aaf9f14041b139f9bb7db2623f_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111312fab2aaf9f14041b139f9bb7db2623f_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,935 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131a6ef7fa4ef247bd9535269125c0846d_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131a6ef7fa4ef247bd9535269125c0846d_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,936 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111328513a16d75849e6819a5dc8afad81cc_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111328513a16d75849e6819a5dc8afad81cc_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,937 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111331ce4e872e974f0286ebef33ce8811c3_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111331ce4e872e974f0286ebef33ce8811c3_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,939 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133792c0c70ac54065b56a5c7b22cdd8a9_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133792c0c70ac54065b56a5c7b22cdd8a9_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,940 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134186331ecff243559145e9af2a0e348b_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134186331ecff243559145e9af2a0e348b_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,942 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134ab669bea23c415bbbe3588cc3387bfd_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134ab669bea23c415bbbe3588cc3387bfd_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,943 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111359cd000915ac46cdb250d916dea525ac_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111359cd000915ac46cdb250d916dea525ac_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,945 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371d299d5a0ce469ebe579d477f7f9f27_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371d299d5a0ce469ebe579d477f7f9f27_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,947 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a409cfa8673f45cbb1aab04a180e8d7f_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a409cfa8673f45cbb1aab04a180e8d7f_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,948 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4a5b97864fa40f6b1a99799f18cb0d3_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4a5b97864fa40f6b1a99799f18cb0d3_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,949 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a5a01fdffc8944cb807c42e38a15042e_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a5a01fdffc8944cb807c42e38a15042e_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,950 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b2fe8f0107f647a29f376248c087420e_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b2fe8f0107f647a29f376248c087420e_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,951 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bfe0463848f042caa537c5a2563e867c_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113bfe0463848f042caa537c5a2563e867c_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,953 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dd85121a0a9b40568144f59dc5179ce4_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dd85121a0a9b40568144f59dc5179ce4_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,962 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ebf692514dba4079a21555ac1f11da3c_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ebf692514dba4079a21555ac1f11da3c_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,964 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f29df27149b24666b71f3e81ac8bdf7d_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f29df27149b24666b71f3e81ac8bdf7d_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,965 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f77d67d4ff5a489bb0ddfed9720a8caf_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f77d67d4ff5a489bb0ddfed9720a8caf_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,966 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fbb335d48ddd4b79b9e2ab68220e110c_0b3f34f131c1584553eed8127d585c45 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113fbb335d48ddd4b79b9e2ab68220e110c_0b3f34f131c1584553eed8127d585c45 2024-11-13T10:40:11,966 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T10:40:11,970 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:11,972 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T10:40:11,974 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T10:40:11,975 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:11,975 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T10:40:11,975 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731494411975"}]},"ts":"9223372036854775807"} 2024-11-13T10:40:11,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T10:40:11,978 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T10:40:11,978 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0b3f34f131c1584553eed8127d585c45, NAME => 'TestAcidGuarantees,,1731494383785.0b3f34f131c1584553eed8127d585c45.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T10:40:11,978 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T10:40:11,978 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731494411978"}]},"ts":"9223372036854775807"} 2024-11-13T10:40:11,980 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T10:40:11,981 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:11,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 113 msec 2024-11-13T10:40:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T10:40:12,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T10:40:12,187 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:12,206 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2019480356_22 at /127.0.0.1:50686 [Waiting for operation #716] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1917562498_22 at /127.0.0.1:52766 [Waiting for operation #863] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1917562498_22 at /127.0.0.1:52832 [Waiting for operation #804] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2019480356_22 at /127.0.0.1:50700 [Waiting for operation #725] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=766 (was 825), ProcessCount=11 (was 11), AvailableMemoryMB=597 (was 905) 2024-11-13T10:40:12,221 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=766, ProcessCount=11, AvailableMemoryMB=597 2024-11-13T10:40:12,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T10:40:12,223 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T10:40:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:12,232 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T10:40:12,233 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:12,234 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-13T10:40:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-13T10:40:12,236 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T10:40:12,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742299_1475 (size=959) 2024-11-13T10:40:12,253 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617 2024-11-13T10:40:12,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742300_1476 (size=53) 2024-11-13T10:40:12,268 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:40:12,269 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing de8553e5e3d4d26fa926e41bd56bc46e, disabling compactions & flushes 2024-11-13T10:40:12,269 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:12,269 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:12,269 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. after waiting 0 ms 2024-11-13T10:40:12,269 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:12,269 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:12,269 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for de8553e5e3d4d26fa926e41bd56bc46e: Waiting for close lock at 1731494412269Disabling compacts and flushes for region at 1731494412269Disabling writes for close at 1731494412269Writing region close event to WAL at 1731494412269Closed at 1731494412269 2024-11-13T10:40:12,271 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T10:40:12,271 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731494412271"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731494412271"}]},"ts":"1731494412271"} 2024-11-13T10:40:12,279 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T10:40:12,280 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T10:40:12,281 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494412281"}]},"ts":"1731494412281"} 2024-11-13T10:40:12,286 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T10:40:12,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de8553e5e3d4d26fa926e41bd56bc46e, ASSIGN}] 2024-11-13T10:40:12,288 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de8553e5e3d4d26fa926e41bd56bc46e, ASSIGN 2024-11-13T10:40:12,289 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de8553e5e3d4d26fa926e41bd56bc46e, ASSIGN; state=OFFLINE, location=770665a7984d,39101,1731494292956; forceNewPlan=false, retain=false 2024-11-13T10:40:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-13T10:40:12,440 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=de8553e5e3d4d26fa926e41bd56bc46e, regionState=OPENING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:40:12,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de8553e5e3d4d26fa926e41bd56bc46e, ASSIGN because future has completed 2024-11-13T10:40:12,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:40:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-13T10:40:12,596 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:12,597 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:40:12,597 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,597 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:40:12,598 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,598 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,599 INFO [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,600 INFO [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de8553e5e3d4d26fa926e41bd56bc46e columnFamilyName A 2024-11-13T10:40:12,600 DEBUG [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:12,601 INFO [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] regionserver.HStore(327): Store=de8553e5e3d4d26fa926e41bd56bc46e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:40:12,601 INFO [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,602 INFO [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de8553e5e3d4d26fa926e41bd56bc46e columnFamilyName B 2024-11-13T10:40:12,602 DEBUG [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:12,603 INFO [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] regionserver.HStore(327): Store=de8553e5e3d4d26fa926e41bd56bc46e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:40:12,603 INFO [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,604 INFO [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de8553e5e3d4d26fa926e41bd56bc46e columnFamilyName C 2024-11-13T10:40:12,604 DEBUG [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:12,604 INFO [StoreOpener-de8553e5e3d4d26fa926e41bd56bc46e-1 {}] regionserver.HStore(327): Store=de8553e5e3d4d26fa926e41bd56bc46e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:40:12,605 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,605 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,606 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,607 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,607 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,607 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:40:12,608 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,611 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T10:40:12,611 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened de8553e5e3d4d26fa926e41bd56bc46e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70683441, jitterRate=0.053265348076820374}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:40:12,611 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,612 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for de8553e5e3d4d26fa926e41bd56bc46e: Running coprocessor pre-open hook at 1731494412598Writing region info on filesystem at 1731494412598Initializing all the Stores at 1731494412598Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494412598Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494412598Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494412599 (+1 ms)Cleaning up temporary data from old regions at 1731494412607 (+8 ms)Running coprocessor post-open hooks at 1731494412611 (+4 ms)Region opened successfully at 1731494412612 (+1 ms) 2024-11-13T10:40:12,613 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., pid=118, masterSystemTime=1731494412593 2024-11-13T10:40:12,615 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:12,615 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:12,615 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=de8553e5e3d4d26fa926e41bd56bc46e, regionState=OPEN, openSeqNum=2, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:40:12,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:40:12,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-13T10:40:12,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 in 175 msec 2024-11-13T10:40:12,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-13T10:40:12,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de8553e5e3d4d26fa926e41bd56bc46e, ASSIGN in 333 msec 2024-11-13T10:40:12,621 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T10:40:12,621 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494412621"}]},"ts":"1731494412621"} 2024-11-13T10:40:12,622 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T10:40:12,623 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T10:40:12,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 400 msec 2024-11-13T10:40:12,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-13T10:40:12,857 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:12,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a364c5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,858 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,858 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,859 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d85ad78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,861 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2155ac5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,862 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,864 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T10:40:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 770665a7984d,44523,1731494292131 2024-11-13T10:40:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4e80da16 2024-11-13T10:40:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T10:40:12,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T10:40:12,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6516064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,867 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22cbc06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,868 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c864a6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,871 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4dad39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,872 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1d26cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,879 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7132c6e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,880 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69054a8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,882 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2092371f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,883 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a095d80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,885 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b0e5af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,886 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638c7afc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,888 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67de41fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,889 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2ffc02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,890 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14a87443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,892 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f2c7d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,893 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c8ab890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,894 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76cbffc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,896 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a22c43f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,897 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e0fe77d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:12,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:12,899 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a0ba452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:12,900 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:12,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6af92961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:12,901 DEBUG [Thread-2123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,903 DEBUG [Thread-2125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,904 DEBUG [Thread-2119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,904 DEBUG [Thread-2132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:40:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:40:12,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:40:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:40:12,913 DEBUG [Thread-2138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,915 DEBUG [Thread-2127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,915 DEBUG [Thread-2121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:40:12,921 DEBUG [Thread-2134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,924 DEBUG [Thread-2136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:40:12,926 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:12,926 DEBUG [Thread-2130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:12,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-13T10:40:12,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,928 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T10:40:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:40:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:40:12,929 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:12,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:12,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:40:12,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2] 2024-11-13T10:40:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:12,961 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:12,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:12,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/80f7a03b4c804231b0e4fb783d561f74 is 50, key is test_row_0/A:col10/1731494412961/Put/seqid=0 2024-11-13T10:40:13,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742301_1477 (size=12001) 2024-11-13T10:40:13,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/80f7a03b4c804231b0e4fb783d561f74 2024-11-13T10:40:13,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494473007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494473015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494473016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494473020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494473020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T10:40:13,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/52bdf49059f5448484a53b016597dfb1 is 50, key is test_row_0/B:col10/1731494412961/Put/seqid=0 2024-11-13T10:40:13,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742302_1478 (size=12001) 2024-11-13T10:40:13,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T10:40:13,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:13,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494473127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494473137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494473138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494473138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494473138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T10:40:13,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:13,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T10:40:13,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494473347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494473356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494473356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494473357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494473357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T10:40:13,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:13,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/52bdf49059f5448484a53b016597dfb1 2024-11-13T10:40:13,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/4b5d000b3c8a40528824d47a321ff36f is 50, key is test_row_0/C:col10/1731494412961/Put/seqid=0 2024-11-13T10:40:13,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742303_1479 (size=12001) 2024-11-13T10:40:13,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T10:40:13,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:13,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T10:40:13,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494473668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494473677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494473677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494473677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494473680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:13,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T10:40:13,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:13,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T10:40:13,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:13,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:13,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:13,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/4b5d000b3c8a40528824d47a321ff36f 2024-11-13T10:40:13,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/80f7a03b4c804231b0e4fb783d561f74 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/80f7a03b4c804231b0e4fb783d561f74 2024-11-13T10:40:13,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/80f7a03b4c804231b0e4fb783d561f74, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T10:40:13,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/52bdf49059f5448484a53b016597dfb1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/52bdf49059f5448484a53b016597dfb1 2024-11-13T10:40:13,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/52bdf49059f5448484a53b016597dfb1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T10:40:13,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/4b5d000b3c8a40528824d47a321ff36f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4b5d000b3c8a40528824d47a321ff36f 2024-11-13T10:40:13,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4b5d000b3c8a40528824d47a321ff36f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-13T10:40:13,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for de8553e5e3d4d26fa926e41bd56bc46e in 930ms, sequenceid=14, compaction requested=false 2024-11-13T10:40:13,892 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T10:40:13,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:14,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T10:40:14,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:14,015 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T10:40:14,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/d91ec386b0cd44c3a71dba53ba3e0581 is 50, key is test_row_0/A:col10/1731494412997/Put/seqid=0 2024-11-13T10:40:14,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742304_1480 (size=12001) 2024-11-13T10:40:14,025 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/d91ec386b0cd44c3a71dba53ba3e0581 2024-11-13T10:40:14,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/c2b34c46362c4a19a722817fe19dad8a is 50, key is test_row_0/B:col10/1731494412997/Put/seqid=0 2024-11-13T10:40:14,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742305_1481 (size=12001) 2024-11-13T10:40:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T10:40:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:14,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:14,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494474215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494474216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494474217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494474217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494474218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494474337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494474338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494474338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494474339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494474339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,443 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/c2b34c46362c4a19a722817fe19dad8a 2024-11-13T10:40:14,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/7be44fa23a834f40b817a489d1f103a5 is 50, key is test_row_0/C:col10/1731494412997/Put/seqid=0 2024-11-13T10:40:14,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742306_1482 (size=12001) 2024-11-13T10:40:14,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494474547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494474547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494474547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494474547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494474557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,864 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/7be44fa23a834f40b817a489d1f103a5 2024-11-13T10:40:14,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/d91ec386b0cd44c3a71dba53ba3e0581 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d91ec386b0cd44c3a71dba53ba3e0581 2024-11-13T10:40:14,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494474867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494474867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494474867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494474867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494474867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:14,878 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d91ec386b0cd44c3a71dba53ba3e0581, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T10:40:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/c2b34c46362c4a19a722817fe19dad8a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/c2b34c46362c4a19a722817fe19dad8a 2024-11-13T10:40:14,882 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/c2b34c46362c4a19a722817fe19dad8a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T10:40:14,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/7be44fa23a834f40b817a489d1f103a5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/7be44fa23a834f40b817a489d1f103a5 2024-11-13T10:40:14,887 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/7be44fa23a834f40b817a489d1f103a5, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T10:40:14,887 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for de8553e5e3d4d26fa926e41bd56bc46e in 872ms, sequenceid=37, compaction requested=false 2024-11-13T10:40:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-13T10:40:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-13T10:40:14,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-13T10:40:14,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9600 sec 2024-11-13T10:40:14,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.9660 sec 2024-11-13T10:40:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T10:40:15,077 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:15,078 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-13T10:40:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T10:40:15,081 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:15,083 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:15,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T10:40:15,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T10:40:15,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:15,237 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T10:40:15,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ed6d149d6cce4af2a991fafe7a1bac3b is 50, key is test_row_0/A:col10/1731494414216/Put/seqid=0 2024-11-13T10:40:15,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742307_1483 (size=12001) 2024-11-13T10:40:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:15,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T10:40:15,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494475479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494475481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494475498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494475499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494475500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,517 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T10:40:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494475608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494475608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494475608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494475627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494475627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,681 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ed6d149d6cce4af2a991fafe7a1bac3b 2024-11-13T10:40:15,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/0c252082363c47d98ad6a03acf75453c is 50, key is test_row_0/B:col10/1731494414216/Put/seqid=0 2024-11-13T10:40:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T10:40:15,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742308_1484 (size=12001) 2024-11-13T10:40:15,745 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/0c252082363c47d98ad6a03acf75453c 2024-11-13T10:40:15,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/d92c40906e3a4078be70a36ca9acba7c is 50, key is test_row_0/C:col10/1731494414216/Put/seqid=0 2024-11-13T10:40:15,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742309_1485 (size=12001) 2024-11-13T10:40:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494475816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494475827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494475827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494475849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494475849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494476147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494476157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494476157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494476167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494476167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,182 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/d92c40906e3a4078be70a36ca9acba7c 2024-11-13T10:40:16,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ed6d149d6cce4af2a991fafe7a1bac3b as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ed6d149d6cce4af2a991fafe7a1bac3b 2024-11-13T10:40:16,210 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ed6d149d6cce4af2a991fafe7a1bac3b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T10:40:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/0c252082363c47d98ad6a03acf75453c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0c252082363c47d98ad6a03acf75453c 2024-11-13T10:40:16,222 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0c252082363c47d98ad6a03acf75453c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T10:40:16,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/d92c40906e3a4078be70a36ca9acba7c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d92c40906e3a4078be70a36ca9acba7c 2024-11-13T10:40:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T10:40:16,231 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d92c40906e3a4078be70a36ca9acba7c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-13T10:40:16,232 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for de8553e5e3d4d26fa926e41bd56bc46e in 995ms, sequenceid=50, compaction requested=true 2024-11-13T10:40:16,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:16,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:16,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-13T10:40:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-13T10:40:16,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-13T10:40:16,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1510 sec 2024-11-13T10:40:16,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1600 sec 2024-11-13T10:40:16,685 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:16,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T10:40:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:16,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/23377cd07ce542a5a0d45d3dcacf42f5 is 50, key is test_row_0/A:col10/1731494416683/Put/seqid=0 2024-11-13T10:40:16,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742310_1486 (size=14341) 2024-11-13T10:40:16,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/23377cd07ce542a5a0d45d3dcacf42f5 2024-11-13T10:40:16,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494476707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/7c78d7b765e44b928e02c5e25e5eda16 is 50, key is test_row_0/B:col10/1731494416683/Put/seqid=0 2024-11-13T10:40:16,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494476709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494476717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494476718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494476719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742311_1487 (size=12001) 2024-11-13T10:40:16,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/7c78d7b765e44b928e02c5e25e5eda16 2024-11-13T10:40:16,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/5c3dffc444cd481eac9d34951919d0ba is 50, key is test_row_0/C:col10/1731494416683/Put/seqid=0 2024-11-13T10:40:16,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742312_1488 (size=12001) 2024-11-13T10:40:16,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/5c3dffc444cd481eac9d34951919d0ba 2024-11-13T10:40:16,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/23377cd07ce542a5a0d45d3dcacf42f5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/23377cd07ce542a5a0d45d3dcacf42f5 2024-11-13T10:40:16,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/23377cd07ce542a5a0d45d3dcacf42f5, entries=200, sequenceid=74, filesize=14.0 K 2024-11-13T10:40:16,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/7c78d7b765e44b928e02c5e25e5eda16 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7c78d7b765e44b928e02c5e25e5eda16 2024-11-13T10:40:16,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7c78d7b765e44b928e02c5e25e5eda16, entries=150, sequenceid=74, filesize=11.7 K 2024-11-13T10:40:16,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/5c3dffc444cd481eac9d34951919d0ba as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/5c3dffc444cd481eac9d34951919d0ba 2024-11-13T10:40:16,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/5c3dffc444cd481eac9d34951919d0ba, entries=150, sequenceid=74, filesize=11.7 K 2024-11-13T10:40:16,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for de8553e5e3d4d26fa926e41bd56bc46e in 153ms, sequenceid=74, compaction requested=true 2024-11-13T10:40:16,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:16,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:16,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:16,839 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:16,839 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:16,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:16,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:16,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:16,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:16,842 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:16,842 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/B is initiating minor compaction (all files) 2024-11-13T10:40:16,842 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/B in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:16,843 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/52bdf49059f5448484a53b016597dfb1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/c2b34c46362c4a19a722817fe19dad8a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0c252082363c47d98ad6a03acf75453c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7c78d7b765e44b928e02c5e25e5eda16] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=46.9 K 2024-11-13T10:40:16,843 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 52bdf49059f5448484a53b016597dfb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731494412914 2024-11-13T10:40:16,843 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:16,843 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/A is initiating minor compaction (all files) 2024-11-13T10:40:16,843 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/A in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:16,844 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/80f7a03b4c804231b0e4fb783d561f74, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d91ec386b0cd44c3a71dba53ba3e0581, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ed6d149d6cce4af2a991fafe7a1bac3b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/23377cd07ce542a5a0d45d3dcacf42f5] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=49.2 K 2024-11-13T10:40:16,844 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b34c46362c4a19a722817fe19dad8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731494412997 2024-11-13T10:40:16,845 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f7a03b4c804231b0e4fb783d561f74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731494412914 2024-11-13T10:40:16,845 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c252082363c47d98ad6a03acf75453c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494414199 2024-11-13T10:40:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:16,845 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d91ec386b0cd44c3a71dba53ba3e0581, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731494412997 2024-11-13T10:40:16,845 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c78d7b765e44b928e02c5e25e5eda16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731494415495 2024-11-13T10:40:16,846 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:16,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T10:40:16,846 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed6d149d6cce4af2a991fafe7a1bac3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494414199 2024-11-13T10:40:16,847 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23377cd07ce542a5a0d45d3dcacf42f5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731494415431 2024-11-13T10:40:16,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/c289fccdf08a49ac84b070bcb3527018 is 50, key is test_row_0/A:col10/1731494416708/Put/seqid=0 2024-11-13T10:40:16,867 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:16,868 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/19c55a17584b491eb66dd930db52bdc9 is 50, key is test_row_0/A:col10/1731494416683/Put/seqid=0 2024-11-13T10:40:16,875 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#B#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:16,875 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/4ee8df0413a94c1a877333ae8d69ffba is 50, key is test_row_0/B:col10/1731494416683/Put/seqid=0 2024-11-13T10:40:16,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742313_1489 (size=16681) 2024-11-13T10:40:16,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742314_1490 (size=12139) 2024-11-13T10:40:16,909 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/19c55a17584b491eb66dd930db52bdc9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/19c55a17584b491eb66dd930db52bdc9 2024-11-13T10:40:16,914 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/A of de8553e5e3d4d26fa926e41bd56bc46e into 19c55a17584b491eb66dd930db52bdc9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:16,914 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:16,914 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/A, priority=12, startTime=1731494416838; duration=0sec 2024-11-13T10:40:16,914 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:16,914 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:A 2024-11-13T10:40:16,914 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:16,916 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:16,916 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/C is initiating minor compaction (all files) 2024-11-13T10:40:16,916 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/C in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:16,917 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4b5d000b3c8a40528824d47a321ff36f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/7be44fa23a834f40b817a489d1f103a5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d92c40906e3a4078be70a36ca9acba7c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/5c3dffc444cd481eac9d34951919d0ba] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=46.9 K 2024-11-13T10:40:16,917 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b5d000b3c8a40528824d47a321ff36f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731494412914 2024-11-13T10:40:16,917 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7be44fa23a834f40b817a489d1f103a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731494412997 2024-11-13T10:40:16,918 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d92c40906e3a4078be70a36ca9acba7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731494414199 2024-11-13T10:40:16,918 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c3dffc444cd481eac9d34951919d0ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731494415495 2024-11-13T10:40:16,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494476902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494476904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494476909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494476915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,931 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494476916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:16,931 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/a7f1344c2b6c47ce99734f78b303332f is 50, key is test_row_0/C:col10/1731494416683/Put/seqid=0 2024-11-13T10:40:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:16,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742315_1491 (size=12139) 2024-11-13T10:40:16,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742316_1492 (size=12139) 2024-11-13T10:40:16,956 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/a7f1344c2b6c47ce99734f78b303332f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/a7f1344c2b6c47ce99734f78b303332f 2024-11-13T10:40:16,962 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/C of de8553e5e3d4d26fa926e41bd56bc46e into a7f1344c2b6c47ce99734f78b303332f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:16,962 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:16,962 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/C, priority=12, startTime=1731494416839; duration=0sec 2024-11-13T10:40:16,962 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:16,963 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:C 2024-11-13T10:40:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494477027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494477027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494477027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494477037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494477037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T10:40:17,238 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:17,239 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-13T10:40:17,242 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T10:40:17,242 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:17,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:17,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494477237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494477237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494477238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494477248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494477248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/c289fccdf08a49ac84b070bcb3527018 2024-11-13T10:40:17,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/9dfe15733efe4ac9b785c159a6051030 is 50, key is test_row_0/B:col10/1731494416708/Put/seqid=0 2024-11-13T10:40:17,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742317_1493 (size=12001) 2024-11-13T10:40:17,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/9dfe15733efe4ac9b785c159a6051030 2024-11-13T10:40:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T10:40:17,349 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/4ee8df0413a94c1a877333ae8d69ffba as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/4ee8df0413a94c1a877333ae8d69ffba 2024-11-13T10:40:17,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/9ae8791f3b1b4142959209eff7dba046 is 50, key is test_row_0/C:col10/1731494416708/Put/seqid=0 2024-11-13T10:40:17,354 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/B of de8553e5e3d4d26fa926e41bd56bc46e into 4ee8df0413a94c1a877333ae8d69ffba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:17,354 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:17,354 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/B, priority=12, startTime=1731494416839; duration=0sec 2024-11-13T10:40:17,354 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:17,354 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:B 2024-11-13T10:40:17,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742318_1494 (size=12001) 2024-11-13T10:40:17,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/9ae8791f3b1b4142959209eff7dba046 2024-11-13T10:40:17,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/c289fccdf08a49ac84b070bcb3527018 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/c289fccdf08a49ac84b070bcb3527018 2024-11-13T10:40:17,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/c289fccdf08a49ac84b070bcb3527018, entries=250, sequenceid=88, filesize=16.3 K 2024-11-13T10:40:17,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/9dfe15733efe4ac9b785c159a6051030 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/9dfe15733efe4ac9b785c159a6051030 2024-11-13T10:40:17,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/9dfe15733efe4ac9b785c159a6051030, entries=150, sequenceid=88, filesize=11.7 K 2024-11-13T10:40:17,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/9ae8791f3b1b4142959209eff7dba046 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/9ae8791f3b1b4142959209eff7dba046 2024-11-13T10:40:17,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/9ae8791f3b1b4142959209eff7dba046, entries=150, sequenceid=88, filesize=11.7 K 2024-11-13T10:40:17,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for de8553e5e3d4d26fa926e41bd56bc46e in 543ms, sequenceid=88, compaction requested=false 2024-11-13T10:40:17,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:17,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T10:40:17,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:17,395 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T10:40:17,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/fddffcadf2ee4174b34fe1ce47de7ac3 is 50, key is test_row_0/A:col10/1731494416913/Put/seqid=0 2024-11-13T10:40:17,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742319_1495 (size=12001) 2024-11-13T10:40:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T10:40:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:17,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:17,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494477597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494477599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494477615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494477617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494477620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494477726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494477726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494477737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494477737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494477737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,807 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/fddffcadf2ee4174b34fe1ce47de7ac3 2024-11-13T10:40:17,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/33eb15fb7b0a4035bf0f7759b35aaed7 is 50, key is test_row_0/B:col10/1731494416913/Put/seqid=0 2024-11-13T10:40:17,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742320_1496 (size=12001) 2024-11-13T10:40:17,825 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/33eb15fb7b0a4035bf0f7759b35aaed7 2024-11-13T10:40:17,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/420912721cb743d898c9647fa4e74d7e is 50, key is test_row_0/C:col10/1731494416913/Put/seqid=0 2024-11-13T10:40:17,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742321_1497 (size=12001) 2024-11-13T10:40:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T10:40:17,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494477947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494477947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494477957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494477958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494477958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:17,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,240 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/420912721cb743d898c9647fa4e74d7e 2024-11-13T10:40:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/fddffcadf2ee4174b34fe1ce47de7ac3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/fddffcadf2ee4174b34fe1ce47de7ac3 2024-11-13T10:40:18,249 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/fddffcadf2ee4174b34fe1ce47de7ac3, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T10:40:18,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/33eb15fb7b0a4035bf0f7759b35aaed7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/33eb15fb7b0a4035bf0f7759b35aaed7 2024-11-13T10:40:18,252 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/33eb15fb7b0a4035bf0f7759b35aaed7, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T10:40:18,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/420912721cb743d898c9647fa4e74d7e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/420912721cb743d898c9647fa4e74d7e 2024-11-13T10:40:18,256 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/420912721cb743d898c9647fa4e74d7e, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T10:40:18,257 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for de8553e5e3d4d26fa926e41bd56bc46e in 862ms, sequenceid=113, compaction requested=true 2024-11-13T10:40:18,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:18,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:18,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-13T10:40:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-13T10:40:18,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-13T10:40:18,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-11-13T10:40:18,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.0270 sec 2024-11-13T10:40:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:18,272 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:18,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T10:40:18,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/8f4660948ced4736b641cc8dcfb247e9 is 50, key is test_row_0/A:col10/1731494417613/Put/seqid=0 2024-11-13T10:40:18,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742322_1498 (size=14341) 2024-11-13T10:40:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494478305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494478318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494478320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494478321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494478322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T10:40:18,384 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:18,386 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-13T10:40:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T10:40:18,389 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:18,394 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:18,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:18,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494478430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494478431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494478447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494478447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494478448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T10:40:18,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T10:40:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:18,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:18,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494478657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494478657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494478666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494478667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494478667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/8f4660948ced4736b641cc8dcfb247e9 2024-11-13T10:40:18,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/293c53eecdf242ac99691ec736d520f2 is 50, key is test_row_0/B:col10/1731494417613/Put/seqid=0 2024-11-13T10:40:18,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T10:40:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:18,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T10:40:18,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742323_1499 (size=12001) 2024-11-13T10:40:18,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T10:40:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:18,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:18,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494478967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494478968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494478989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494478989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494478991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T10:40:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:19,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T10:40:19,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/293c53eecdf242ac99691ec736d520f2 2024-11-13T10:40:19,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/4214ad9d77a9442fa1b1041ae6c6aaaa is 50, key is test_row_0/C:col10/1731494417613/Put/seqid=0 2024-11-13T10:40:19,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T10:40:19,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:19,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:19,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:19,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742324_1500 (size=12001) 2024-11-13T10:40:19,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/4214ad9d77a9442fa1b1041ae6c6aaaa 2024-11-13T10:40:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/8f4660948ced4736b641cc8dcfb247e9 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/8f4660948ced4736b641cc8dcfb247e9 2024-11-13T10:40:19,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/8f4660948ced4736b641cc8dcfb247e9, entries=200, sequenceid=128, filesize=14.0 K 2024-11-13T10:40:19,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/293c53eecdf242ac99691ec736d520f2 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/293c53eecdf242ac99691ec736d520f2 2024-11-13T10:40:19,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/293c53eecdf242ac99691ec736d520f2, entries=150, sequenceid=128, filesize=11.7 K 2024-11-13T10:40:19,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/4214ad9d77a9442fa1b1041ae6c6aaaa as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4214ad9d77a9442fa1b1041ae6c6aaaa 2024-11-13T10:40:19,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4214ad9d77a9442fa1b1041ae6c6aaaa, entries=150, sequenceid=128, filesize=11.7 K 2024-11-13T10:40:19,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for de8553e5e3d4d26fa926e41bd56bc46e in 955ms, sequenceid=128, compaction requested=true 2024-11-13T10:40:19,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:19,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:19,228 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:19,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:19,228 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:19,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:19,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:19,229 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55162 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:19,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:19,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:19,230 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/A is initiating minor compaction (all files) 2024-11-13T10:40:19,230 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:19,230 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/A in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:19,230 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/B is initiating minor compaction (all files) 2024-11-13T10:40:19,230 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/19c55a17584b491eb66dd930db52bdc9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/c289fccdf08a49ac84b070bcb3527018, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/fddffcadf2ee4174b34fe1ce47de7ac3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/8f4660948ced4736b641cc8dcfb247e9] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=53.9 K 2024-11-13T10:40:19,230 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/B in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:19,230 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/4ee8df0413a94c1a877333ae8d69ffba, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/9dfe15733efe4ac9b785c159a6051030, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/33eb15fb7b0a4035bf0f7759b35aaed7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/293c53eecdf242ac99691ec736d520f2] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=47.0 K 2024-11-13T10:40:19,230 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19c55a17584b491eb66dd930db52bdc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731494415495 2024-11-13T10:40:19,230 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ee8df0413a94c1a877333ae8d69ffba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731494415495 2024-11-13T10:40:19,230 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c289fccdf08a49ac84b070bcb3527018, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731494416708 2024-11-13T10:40:19,231 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dfe15733efe4ac9b785c159a6051030, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731494416708 2024-11-13T10:40:19,231 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting fddffcadf2ee4174b34fe1ce47de7ac3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731494416907 2024-11-13T10:40:19,231 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 33eb15fb7b0a4035bf0f7759b35aaed7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731494416907 2024-11-13T10:40:19,231 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f4660948ced4736b641cc8dcfb247e9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731494417611 2024-11-13T10:40:19,231 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 293c53eecdf242ac99691ec736d520f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731494417611 2024-11-13T10:40:19,251 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:19,252 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/d8e616d148d5445590db49f2c2ca1072 is 50, key is test_row_0/A:col10/1731494417613/Put/seqid=0 2024-11-13T10:40:19,259 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#B#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:19,259 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/5bd7107a4e2a4c4e9a31ff7dc556d542 is 50, key is test_row_0/B:col10/1731494417613/Put/seqid=0 2024-11-13T10:40:19,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742325_1501 (size=12275) 2024-11-13T10:40:19,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742326_1502 (size=12275) 2024-11-13T10:40:19,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T10:40:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:19,336 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T10:40:19,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/472f045f48334520853e0fee1bcd038d is 50, key is test_row_0/A:col10/1731494418320/Put/seqid=0 2024-11-13T10:40:19,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742327_1503 (size=12151) 2024-11-13T10:40:19,423 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/472f045f48334520853e0fee1bcd038d 2024-11-13T10:40:19,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/34946a51ed044f91b1a2069beac725a7 is 50, key is test_row_0/B:col10/1731494418320/Put/seqid=0 2024-11-13T10:40:19,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742328_1504 (size=12151) 2024-11-13T10:40:19,484 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/34946a51ed044f91b1a2069beac725a7 2024-11-13T10:40:19,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/113f70a26cb74b6f8bb96269a1cbcada is 50, key is test_row_0/C:col10/1731494418320/Put/seqid=0 2024-11-13T10:40:19,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:19,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742329_1505 (size=12151) 2024-11-13T10:40:19,518 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/113f70a26cb74b6f8bb96269a1cbcada 2024-11-13T10:40:19,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/472f045f48334520853e0fee1bcd038d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/472f045f48334520853e0fee1bcd038d 2024-11-13T10:40:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T10:40:19,531 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/472f045f48334520853e0fee1bcd038d, entries=150, sequenceid=149, filesize=11.9 K 2024-11-13T10:40:19,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/34946a51ed044f91b1a2069beac725a7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/34946a51ed044f91b1a2069beac725a7 2024-11-13T10:40:19,538 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/34946a51ed044f91b1a2069beac725a7, entries=150, sequenceid=149, filesize=11.9 K 2024-11-13T10:40:19,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/113f70a26cb74b6f8bb96269a1cbcada as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/113f70a26cb74b6f8bb96269a1cbcada 2024-11-13T10:40:19,543 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/113f70a26cb74b6f8bb96269a1cbcada, entries=150, sequenceid=149, filesize=11.9 K 2024-11-13T10:40:19,554 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=20.13 KB/20610 for de8553e5e3d4d26fa926e41bd56bc46e in 218ms, sequenceid=149, compaction requested=true 2024-11-13T10:40:19,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:19,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:19,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-13T10:40:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-13T10:40:19,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-13T10:40:19,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1620 sec 2024-11-13T10:40:19,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.1720 sec 2024-11-13T10:40:19,576 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:19,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:19,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ac502e5c413640f6aa6baec10847e505 is 50, key is test_row_0/A:col10/1731494419573/Put/seqid=0 2024-11-13T10:40:19,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742330_1506 (size=14541) 2024-11-13T10:40:19,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ac502e5c413640f6aa6baec10847e505 2024-11-13T10:40:19,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/7503495d41f84e4c8145ab3ee52b2d50 is 50, key is test_row_0/B:col10/1731494419573/Put/seqid=0 2024-11-13T10:40:19,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742331_1507 (size=12151) 2024-11-13T10:40:19,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/7503495d41f84e4c8145ab3ee52b2d50 2024-11-13T10:40:19,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/0bde1417b0644f93a8de36095793ca62 is 50, key is test_row_0/C:col10/1731494419573/Put/seqid=0 2024-11-13T10:40:19,712 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/d8e616d148d5445590db49f2c2ca1072 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d8e616d148d5445590db49f2c2ca1072 2024-11-13T10:40:19,727 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/A of de8553e5e3d4d26fa926e41bd56bc46e into d8e616d148d5445590db49f2c2ca1072(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:19,727 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:19,727 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/A, priority=12, startTime=1731494419228; duration=0sec 2024-11-13T10:40:19,727 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:19,727 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:A 2024-11-13T10:40:19,727 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T10:40:19,729 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60293 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T10:40:19,729 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/C is initiating minor compaction (all files) 2024-11-13T10:40:19,729 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/C in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:19,730 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/a7f1344c2b6c47ce99734f78b303332f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/9ae8791f3b1b4142959209eff7dba046, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/420912721cb743d898c9647fa4e74d7e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4214ad9d77a9442fa1b1041ae6c6aaaa, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/113f70a26cb74b6f8bb96269a1cbcada] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=58.9 K 2024-11-13T10:40:19,730 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7f1344c2b6c47ce99734f78b303332f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731494415495 2024-11-13T10:40:19,730 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/5bd7107a4e2a4c4e9a31ff7dc556d542 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5bd7107a4e2a4c4e9a31ff7dc556d542 2024-11-13T10:40:19,730 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ae8791f3b1b4142959209eff7dba046, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731494416708 2024-11-13T10:40:19,731 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 420912721cb743d898c9647fa4e74d7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731494416907 2024-11-13T10:40:19,731 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4214ad9d77a9442fa1b1041ae6c6aaaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731494417611 2024-11-13T10:40:19,732 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 113f70a26cb74b6f8bb96269a1cbcada, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731494418300 2024-11-13T10:40:19,737 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/B of de8553e5e3d4d26fa926e41bd56bc46e into 5bd7107a4e2a4c4e9a31ff7dc556d542(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:19,737 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:19,737 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/B, priority=12, startTime=1731494419228; duration=0sec 2024-11-13T10:40:19,737 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:19,737 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:B 2024-11-13T10:40:19,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742332_1508 (size=12151) 2024-11-13T10:40:19,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494479706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494479706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494479708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494479709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494479715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,758 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#C#compaction#428 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:19,758 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/84a6126811754792a8d38b5dc04532be is 50, key is test_row_0/C:col10/1731494418320/Put/seqid=0 2024-11-13T10:40:19,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742333_1509 (size=12459) 2024-11-13T10:40:19,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494479857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494479858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494479858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494479858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:19,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494479860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494480076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494480087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494480087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494480087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494480087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/0bde1417b0644f93a8de36095793ca62 2024-11-13T10:40:20,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ac502e5c413640f6aa6baec10847e505 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ac502e5c413640f6aa6baec10847e505 2024-11-13T10:40:20,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ac502e5c413640f6aa6baec10847e505, entries=200, sequenceid=161, filesize=14.2 K 2024-11-13T10:40:20,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/7503495d41f84e4c8145ab3ee52b2d50 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7503495d41f84e4c8145ab3ee52b2d50 2024-11-13T10:40:20,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7503495d41f84e4c8145ab3ee52b2d50, entries=150, sequenceid=161, filesize=11.9 K 2024-11-13T10:40:20,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/0bde1417b0644f93a8de36095793ca62 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0bde1417b0644f93a8de36095793ca62 2024-11-13T10:40:20,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0bde1417b0644f93a8de36095793ca62, entries=150, sequenceid=161, filesize=11.9 K 2024-11-13T10:40:20,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for de8553e5e3d4d26fa926e41bd56bc46e in 602ms, sequenceid=161, compaction requested=true 2024-11-13T10:40:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:20,178 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:A, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:20,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:B, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:20,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:20,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:20,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:40:20,179 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38967 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:20,179 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/A is initiating minor compaction (all files) 2024-11-13T10:40:20,179 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/A in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:20,179 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d8e616d148d5445590db49f2c2ca1072, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/472f045f48334520853e0fee1bcd038d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ac502e5c413640f6aa6baec10847e505] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=38.1 K 2024-11-13T10:40:20,180 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting d8e616d148d5445590db49f2c2ca1072, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731494417611 2024-11-13T10:40:20,180 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 472f045f48334520853e0fee1bcd038d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731494418300 2024-11-13T10:40:20,180 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ac502e5c413640f6aa6baec10847e505, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494419550 2024-11-13T10:40:20,186 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#A#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:20,186 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/4b65f3529c6c4cbda1614465b06b366f is 50, key is test_row_0/A:col10/1731494419573/Put/seqid=0 2024-11-13T10:40:20,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742334_1510 (size=12527) 2024-11-13T10:40:20,201 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/84a6126811754792a8d38b5dc04532be as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/84a6126811754792a8d38b5dc04532be 2024-11-13T10:40:20,206 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/C of de8553e5e3d4d26fa926e41bd56bc46e into 84a6126811754792a8d38b5dc04532be(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:20,206 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:20,206 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/C, priority=11, startTime=1731494419229; duration=0sec 2024-11-13T10:40:20,206 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:40:20,206 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:C 2024-11-13T10:40:20,206 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:C 2024-11-13T10:40:20,206 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:20,207 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:20,207 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/B is initiating minor compaction (all files) 2024-11-13T10:40:20,207 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/B in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:20,207 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5bd7107a4e2a4c4e9a31ff7dc556d542, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/34946a51ed044f91b1a2069beac725a7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7503495d41f84e4c8145ab3ee52b2d50] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=35.7 K 2024-11-13T10:40:20,208 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bd7107a4e2a4c4e9a31ff7dc556d542, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731494417611 2024-11-13T10:40:20,208 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34946a51ed044f91b1a2069beac725a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731494418300 2024-11-13T10:40:20,209 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7503495d41f84e4c8145ab3ee52b2d50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494419550 2024-11-13T10:40:20,216 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#B#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:20,216 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/6deb60734e83467f96c27eb9fbf3cf37 is 50, key is test_row_0/B:col10/1731494419573/Put/seqid=0 2024-11-13T10:40:20,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742335_1511 (size=12527) 2024-11-13T10:40:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:20,401 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:20,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:40:20,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ca1300a7f99947119d14a79b725280e0 is 50, key is test_row_0/A:col10/1731494419707/Put/seqid=0 2024-11-13T10:40:20,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742336_1512 (size=14541) 2024-11-13T10:40:20,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494480416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494480418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494480418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494480423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494480424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494480537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494480537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T10:40:20,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494480537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,547 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,548 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-13T10:40:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T10:40:20,550 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:20,551 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:20,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:20,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494480547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494480547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,598 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/4b65f3529c6c4cbda1614465b06b366f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/4b65f3529c6c4cbda1614465b06b366f 2024-11-13T10:40:20,604 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/A of de8553e5e3d4d26fa926e41bd56bc46e into 4b65f3529c6c4cbda1614465b06b366f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:20,604 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:20,604 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/A, priority=13, startTime=1731494420178; duration=0sec 2024-11-13T10:40:20,604 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:20,604 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:A 2024-11-13T10:40:20,604 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T10:40:20,604 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T10:40:20,605 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T10:40:20,605 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. because compaction request was cancelled 2024-11-13T10:40:20,605 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:C 2024-11-13T10:40:20,636 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/6deb60734e83467f96c27eb9fbf3cf37 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/6deb60734e83467f96c27eb9fbf3cf37 2024-11-13T10:40:20,641 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/B of de8553e5e3d4d26fa926e41bd56bc46e into 6deb60734e83467f96c27eb9fbf3cf37(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:20,641 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:20,641 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/B, priority=13, startTime=1731494420178; duration=0sec 2024-11-13T10:40:20,641 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:20,641 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:B 2024-11-13T10:40:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T10:40:20,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T10:40:20,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:20,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:20,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:20,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:20,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:20,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494480757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494480757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494480757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494480767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494480767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:20,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ca1300a7f99947119d14a79b725280e0 2024-11-13T10:40:20,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/5ebbe62845144786a3cccf6939a77fde is 50, key is test_row_0/B:col10/1731494419707/Put/seqid=0 2024-11-13T10:40:20,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T10:40:20,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:20,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:20,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:20,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T10:40:20,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742337_1513 (size=12151) 2024-11-13T10:40:21,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T10:40:21,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494481077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:21,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494481077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:21,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494481077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:21,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:21,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494481086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:21,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494481086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:21,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T10:40:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T10:40:21,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/5ebbe62845144786a3cccf6939a77fde 2024-11-13T10:40:21,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/d669e25f59bf44a3b01f003dd2c9e136 is 50, key is test_row_0/C:col10/1731494419707/Put/seqid=0 2024-11-13T10:40:21,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742338_1514 (size=12151) 2024-11-13T10:40:21,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T10:40:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T10:40:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:21,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494481588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:21,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494481597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:21,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494481599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:21,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494481607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494481607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:21,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T10:40:21,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:21,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:21,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/d669e25f59bf44a3b01f003dd2c9e136 2024-11-13T10:40:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T10:40:21,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ca1300a7f99947119d14a79b725280e0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ca1300a7f99947119d14a79b725280e0 2024-11-13T10:40:21,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ca1300a7f99947119d14a79b725280e0, entries=200, sequenceid=189, filesize=14.2 K 2024-11-13T10:40:21,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/5ebbe62845144786a3cccf6939a77fde as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5ebbe62845144786a3cccf6939a77fde 2024-11-13T10:40:21,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5ebbe62845144786a3cccf6939a77fde, entries=150, sequenceid=189, filesize=11.9 K 2024-11-13T10:40:21,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/d669e25f59bf44a3b01f003dd2c9e136 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d669e25f59bf44a3b01f003dd2c9e136 2024-11-13T10:40:21,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d669e25f59bf44a3b01f003dd2c9e136, entries=150, sequenceid=189, filesize=11.9 K 2024-11-13T10:40:21,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for de8553e5e3d4d26fa926e41bd56bc46e in 1308ms, sequenceid=189, compaction requested=true 2024-11-13T10:40:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:21,709 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T10:40:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:21,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:40:21,710 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T10:40:21,710 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T10:40:21,710 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. because compaction request was cancelled 2024-11-13T10:40:21,710 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:A 2024-11-13T10:40:21,710 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:21,710 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T10:40:21,710 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:21,711 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/C is initiating minor compaction (all files) 2024-11-13T10:40:21,711 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/C in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,711 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/84a6126811754792a8d38b5dc04532be, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0bde1417b0644f93a8de36095793ca62, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d669e25f59bf44a3b01f003dd2c9e136] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=35.9 K 2024-11-13T10:40:21,711 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84a6126811754792a8d38b5dc04532be, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731494418300 2024-11-13T10:40:21,711 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bde1417b0644f93a8de36095793ca62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494419550 2024-11-13T10:40:21,712 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d669e25f59bf44a3b01f003dd2c9e136, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731494419705 2024-11-13T10:40:21,712 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-13T10:40:21,713 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-13T10:40:21,713 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. because compaction request was cancelled 2024-11-13T10:40:21,713 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:B 2024-11-13T10:40:21,739 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:21,740 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/0ffc4a2442c34afe9845b519319ac897 is 50, key is test_row_0/C:col10/1731494419707/Put/seqid=0 2024-11-13T10:40:21,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742339_1515 (size=12561) 2024-11-13T10:40:21,783 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/0ffc4a2442c34afe9845b519319ac897 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0ffc4a2442c34afe9845b519319ac897 2024-11-13T10:40:21,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T10:40:21,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:21,792 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:21,792 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/C of de8553e5e3d4d26fa926e41bd56bc46e into 0ffc4a2442c34afe9845b519319ac897(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:21,792 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:21,792 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/C, priority=13, startTime=1731494421709; duration=0sec 2024-11-13T10:40:21,792 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:21,792 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:C 2024-11-13T10:40:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/38b5a951f73d4a078c27cf7bae76f437 is 50, key is test_row_0/A:col10/1731494420415/Put/seqid=0 2024-11-13T10:40:21,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742340_1516 (size=12151) 2024-11-13T10:40:21,812 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/38b5a951f73d4a078c27cf7bae76f437 2024-11-13T10:40:21,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/b1719abf1881419e809fcfab330694f5 is 50, key is test_row_0/B:col10/1731494420415/Put/seqid=0 2024-11-13T10:40:21,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742341_1517 (size=12151) 2024-11-13T10:40:21,835 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/b1719abf1881419e809fcfab330694f5 2024-11-13T10:40:21,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/993e5d95d4234eb6b7d1e6a910da0d63 is 50, key is test_row_0/C:col10/1731494420415/Put/seqid=0 2024-11-13T10:40:21,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742342_1518 (size=12151) 2024-11-13T10:40:22,281 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/993e5d95d4234eb6b7d1e6a910da0d63 2024-11-13T10:40:22,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/38b5a951f73d4a078c27cf7bae76f437 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/38b5a951f73d4a078c27cf7bae76f437 2024-11-13T10:40:22,307 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/38b5a951f73d4a078c27cf7bae76f437, entries=150, sequenceid=203, filesize=11.9 K 2024-11-13T10:40:22,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/b1719abf1881419e809fcfab330694f5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/b1719abf1881419e809fcfab330694f5 2024-11-13T10:40:22,325 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/b1719abf1881419e809fcfab330694f5, entries=150, sequenceid=203, filesize=11.9 K 2024-11-13T10:40:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/993e5d95d4234eb6b7d1e6a910da0d63 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/993e5d95d4234eb6b7d1e6a910da0d63 2024-11-13T10:40:22,330 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/993e5d95d4234eb6b7d1e6a910da0d63, entries=150, sequenceid=203, filesize=11.9 K 2024-11-13T10:40:22,331 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for de8553e5e3d4d26fa926e41bd56bc46e in 540ms, sequenceid=203, compaction requested=true 2024-11-13T10:40:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:22,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-13T10:40:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-13T10:40:22,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-13T10:40:22,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7810 sec 2024-11-13T10:40:22,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7870 sec 2024-11-13T10:40:22,631 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:22,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:22,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/0cab8a0b06c64ef88f9dc1dc650fe250 is 50, key is test_row_0/A:col10/1731494422627/Put/seqid=0 2024-11-13T10:40:22,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742343_1519 (size=21715) 2024-11-13T10:40:22,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/0cab8a0b06c64ef88f9dc1dc650fe250 2024-11-13T10:40:22,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/1df8be75f856474187e1f35331b3fb99 is 50, key is test_row_0/B:col10/1731494422627/Put/seqid=0 2024-11-13T10:40:22,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494482675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:22,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742344_1520 (size=12151) 2024-11-13T10:40:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T10:40:22,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494482686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:22,697 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:22,698 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-13T10:40:22,700 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T10:40:22,701 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:22,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:22,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494482689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:22,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494482691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:22,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494482701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:22,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494482797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:22,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T10:40:22,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494482807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:22,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494482816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494482816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:22,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494482826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:22,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T10:40:22,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:22,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:22,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:22,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:22,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:23,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494483007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T10:40:23,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:23,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:23,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:23,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:23,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T10:40:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:23,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494483026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494483026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494483039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494483047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/1df8be75f856474187e1f35331b3fb99 2024-11-13T10:40:23,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/afd308abba5249b88bf1b4ed2a28c829 is 50, key is test_row_0/C:col10/1731494422627/Put/seqid=0 2024-11-13T10:40:23,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742345_1521 (size=12151) 2024-11-13T10:40:23,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/afd308abba5249b88bf1b4ed2a28c829 2024-11-13T10:40:23,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/0cab8a0b06c64ef88f9dc1dc650fe250 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/0cab8a0b06c64ef88f9dc1dc650fe250 2024-11-13T10:40:23,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/0cab8a0b06c64ef88f9dc1dc650fe250, entries=350, sequenceid=215, filesize=21.2 K 2024-11-13T10:40:23,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/1df8be75f856474187e1f35331b3fb99 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1df8be75f856474187e1f35331b3fb99 2024-11-13T10:40:23,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1df8be75f856474187e1f35331b3fb99, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T10:40:23,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T10:40:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:23,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:23,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/afd308abba5249b88bf1b4ed2a28c829 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/afd308abba5249b88bf1b4ed2a28c829 2024-11-13T10:40:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:23,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/afd308abba5249b88bf1b4ed2a28c829, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T10:40:23,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for de8553e5e3d4d26fa926e41bd56bc46e in 584ms, sequenceid=215, compaction requested=true 2024-11-13T10:40:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:40:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T10:40:23,218 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:23,220 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:23,222 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60934 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:23,222 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/A is initiating minor compaction (all files) 2024-11-13T10:40:23,222 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/A in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:23,222 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/4b65f3529c6c4cbda1614465b06b366f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ca1300a7f99947119d14a79b725280e0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/38b5a951f73d4a078c27cf7bae76f437, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/0cab8a0b06c64ef88f9dc1dc650fe250] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=59.5 K 2024-11-13T10:40:23,226 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:23,226 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/B is initiating minor compaction (all files) 2024-11-13T10:40:23,226 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/B in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:23,226 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/6deb60734e83467f96c27eb9fbf3cf37, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5ebbe62845144786a3cccf6939a77fde, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/b1719abf1881419e809fcfab330694f5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1df8be75f856474187e1f35331b3fb99] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=47.8 K 2024-11-13T10:40:23,226 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b65f3529c6c4cbda1614465b06b366f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494419550 2024-11-13T10:40:23,232 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6deb60734e83467f96c27eb9fbf3cf37, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494419550 2024-11-13T10:40:23,232 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1300a7f99947119d14a79b725280e0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731494419673 2024-11-13T10:40:23,233 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ebbe62845144786a3cccf6939a77fde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731494419705 2024-11-13T10:40:23,233 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b5a951f73d4a078c27cf7bae76f437, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731494420415 2024-11-13T10:40:23,233 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1719abf1881419e809fcfab330694f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731494420415 2024-11-13T10:40:23,236 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cab8a0b06c64ef88f9dc1dc650fe250, keycount=350, bloomtype=ROW, size=21.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731494422619 2024-11-13T10:40:23,237 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1df8be75f856474187e1f35331b3fb99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731494422626 2024-11-13T10:40:23,275 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#B#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:23,276 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a5ff9e9ab08447dbb8989f976a87bdcc is 50, key is test_row_0/B:col10/1731494422627/Put/seqid=0 2024-11-13T10:40:23,286 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#A#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:23,287 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/b00cc834fd2d4c48bd8c9165e9175f0c is 50, key is test_row_0/A:col10/1731494422627/Put/seqid=0 2024-11-13T10:40:23,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742347_1523 (size=12663) 2024-11-13T10:40:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T10:40:23,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T10:40:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:23,343 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T10:40:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:23,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:23,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742346_1522 (size=12663) 2024-11-13T10:40:23,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/a53774e7c14f4b59a4dba841d8c2556c is 50, key is test_row_0/A:col10/1731494422689/Put/seqid=0 2024-11-13T10:40:23,368 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a5ff9e9ab08447dbb8989f976a87bdcc as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a5ff9e9ab08447dbb8989f976a87bdcc 2024-11-13T10:40:23,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742348_1524 (size=14541) 2024-11-13T10:40:23,372 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/a53774e7c14f4b59a4dba841d8c2556c 2024-11-13T10:40:23,375 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/B of de8553e5e3d4d26fa926e41bd56bc46e into a5ff9e9ab08447dbb8989f976a87bdcc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:23,375 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:23,375 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/B, priority=12, startTime=1731494423217; duration=0sec 2024-11-13T10:40:23,375 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:23,375 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:B 2024-11-13T10:40:23,375 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:23,377 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:23,377 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/C is initiating minor compaction (all files) 2024-11-13T10:40:23,377 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/C in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:23,377 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0ffc4a2442c34afe9845b519319ac897, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/993e5d95d4234eb6b7d1e6a910da0d63, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/afd308abba5249b88bf1b4ed2a28c829] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=36.0 K 2024-11-13T10:40:23,378 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ffc4a2442c34afe9845b519319ac897, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731494419705 2024-11-13T10:40:23,378 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 993e5d95d4234eb6b7d1e6a910da0d63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731494420415 2024-11-13T10:40:23,379 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd308abba5249b88bf1b4ed2a28c829, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731494422626 2024-11-13T10:40:23,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/1cb2aed3519849eaadacccd91888c041 is 50, key is test_row_0/B:col10/1731494422689/Put/seqid=0 2024-11-13T10:40:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494483368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,387 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:23,388 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/b04fa82d9e2f482ab46b1abfaac8bb48 is 50, key is test_row_0/C:col10/1731494422627/Put/seqid=0 2024-11-13T10:40:23,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742349_1525 (size=12151) 2024-11-13T10:40:23,390 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/1cb2aed3519849eaadacccd91888c041 2024-11-13T10:40:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494483378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494483379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494483381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494483383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/356cb0de2ed54366bae445ee9d971d83 is 50, key is test_row_0/C:col10/1731494422689/Put/seqid=0 2024-11-13T10:40:23,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742350_1526 (size=12663) 2024-11-13T10:40:23,422 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/b04fa82d9e2f482ab46b1abfaac8bb48 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/b04fa82d9e2f482ab46b1abfaac8bb48 2024-11-13T10:40:23,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742351_1527 (size=12151) 2024-11-13T10:40:23,439 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/C of de8553e5e3d4d26fa926e41bd56bc46e into b04fa82d9e2f482ab46b1abfaac8bb48(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:23,439 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:23,439 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/C, priority=13, startTime=1731494423217; duration=0sec 2024-11-13T10:40:23,439 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:23,439 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:C 2024-11-13T10:40:23,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494483487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494483507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494483507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494483507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494483507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494483707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494483726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494483727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494483727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494483728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:23,740 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/b00cc834fd2d4c48bd8c9165e9175f0c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/b00cc834fd2d4c48bd8c9165e9175f0c 2024-11-13T10:40:23,747 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/A of de8553e5e3d4d26fa926e41bd56bc46e into b00cc834fd2d4c48bd8c9165e9175f0c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:23,747 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:23,747 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/A, priority=12, startTime=1731494423217; duration=0sec 2024-11-13T10:40:23,747 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:23,747 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:A 2024-11-13T10:40:23,835 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/356cb0de2ed54366bae445ee9d971d83 2024-11-13T10:40:23,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/a53774e7c14f4b59a4dba841d8c2556c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a53774e7c14f4b59a4dba841d8c2556c 2024-11-13T10:40:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T10:40:23,851 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a53774e7c14f4b59a4dba841d8c2556c, entries=200, sequenceid=240, filesize=14.2 K 2024-11-13T10:40:23,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/1cb2aed3519849eaadacccd91888c041 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1cb2aed3519849eaadacccd91888c041 2024-11-13T10:40:23,857 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1cb2aed3519849eaadacccd91888c041, entries=150, sequenceid=240, filesize=11.9 K 2024-11-13T10:40:23,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/356cb0de2ed54366bae445ee9d971d83 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/356cb0de2ed54366bae445ee9d971d83 2024-11-13T10:40:23,872 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/356cb0de2ed54366bae445ee9d971d83, entries=150, sequenceid=240, filesize=11.9 K 2024-11-13T10:40:23,873 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for de8553e5e3d4d26fa926e41bd56bc46e in 530ms, sequenceid=240, compaction requested=false 2024-11-13T10:40:23,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:23,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:23,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-13T10:40:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-13T10:40:23,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-13T10:40:23,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1730 sec 2024-11-13T10:40:23,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1810 sec 2024-11-13T10:40:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:24,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:24,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T10:40:24,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/a510552f0d7a467cbfb8bf5eed32a457 is 50, key is test_row_0/A:col10/1731494423375/Put/seqid=0 2024-11-13T10:40:24,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742352_1528 (size=14541) 2024-11-13T10:40:24,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494484134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494484149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494484151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494484156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494484156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494484257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494484276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494484276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494484277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494484277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/a510552f0d7a467cbfb8bf5eed32a457 2024-11-13T10:40:24,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/ca88faed8feb4296aa5d0787b67032a7 is 50, key is test_row_0/B:col10/1731494423375/Put/seqid=0 2024-11-13T10:40:24,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742353_1529 (size=12151) 2024-11-13T10:40:24,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/ca88faed8feb4296aa5d0787b67032a7 2024-11-13T10:40:24,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494484480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/ee61b13e964b4ad7a117fa6b59ab9a84 is 50, key is test_row_0/C:col10/1731494423375/Put/seqid=0 2024-11-13T10:40:24,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494484489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494484489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494484489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494484501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742354_1530 (size=12151) 2024-11-13T10:40:24,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/ee61b13e964b4ad7a117fa6b59ab9a84 2024-11-13T10:40:24,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/a510552f0d7a467cbfb8bf5eed32a457 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a510552f0d7a467cbfb8bf5eed32a457 2024-11-13T10:40:24,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a510552f0d7a467cbfb8bf5eed32a457, entries=200, sequenceid=255, filesize=14.2 K 2024-11-13T10:40:24,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/ca88faed8feb4296aa5d0787b67032a7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/ca88faed8feb4296aa5d0787b67032a7 2024-11-13T10:40:24,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/ca88faed8feb4296aa5d0787b67032a7, entries=150, sequenceid=255, filesize=11.9 K 2024-11-13T10:40:24,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/ee61b13e964b4ad7a117fa6b59ab9a84 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/ee61b13e964b4ad7a117fa6b59ab9a84 2024-11-13T10:40:24,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/ee61b13e964b4ad7a117fa6b59ab9a84, entries=150, sequenceid=255, filesize=11.9 K 2024-11-13T10:40:24,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for de8553e5e3d4d26fa926e41bd56bc46e in 565ms, sequenceid=255, compaction requested=true 2024-11-13T10:40:24,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:24,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:24,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:24,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:24,599 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:24,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:24,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:24,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:24,599 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:24,604 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:24,604 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/A is initiating minor compaction (all files) 2024-11-13T10:40:24,604 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/A in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:24,604 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/b00cc834fd2d4c48bd8c9165e9175f0c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a53774e7c14f4b59a4dba841d8c2556c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a510552f0d7a467cbfb8bf5eed32a457] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=40.8 K 2024-11-13T10:40:24,608 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:24,608 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b00cc834fd2d4c48bd8c9165e9175f0c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731494422626 2024-11-13T10:40:24,609 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/B is initiating minor compaction (all files) 2024-11-13T10:40:24,609 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/B in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:24,609 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a5ff9e9ab08447dbb8989f976a87bdcc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1cb2aed3519849eaadacccd91888c041, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/ca88faed8feb4296aa5d0787b67032a7] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=36.1 K 2024-11-13T10:40:24,609 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a53774e7c14f4b59a4dba841d8c2556c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731494422680 2024-11-13T10:40:24,609 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting a5ff9e9ab08447dbb8989f976a87bdcc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731494422626 2024-11-13T10:40:24,609 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a510552f0d7a467cbfb8bf5eed32a457, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731494423364 2024-11-13T10:40:24,610 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb2aed3519849eaadacccd91888c041, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731494422688 2024-11-13T10:40:24,610 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ca88faed8feb4296aa5d0787b67032a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731494423375 2024-11-13T10:40:24,627 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#A#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:24,628 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/1bb3999613b646969a8a1d45444cb36e is 50, key is test_row_0/A:col10/1731494423375/Put/seqid=0 2024-11-13T10:40:24,634 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#B#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:24,635 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/020060fdcc4c47c8ac59f4300e5a37bb is 50, key is test_row_0/B:col10/1731494423375/Put/seqid=0 2024-11-13T10:40:24,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742356_1532 (size=12765) 2024-11-13T10:40:24,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742355_1531 (size=12765) 2024-11-13T10:40:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:24,811 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:24,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:40:24,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/a3ba9b25eafe47bf944b16d4785205e3 is 50, key is test_row_0/A:col10/1731494424149/Put/seqid=0 2024-11-13T10:40:24,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742357_1533 (size=14741) 2024-11-13T10:40:24,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/a3ba9b25eafe47bf944b16d4785205e3 2024-11-13T10:40:24,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/99cb0c5e9ee9429db7cc19ddc4cf55ff is 50, key is test_row_0/B:col10/1731494424149/Put/seqid=0 2024-11-13T10:40:24,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494484828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494484831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494484833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494484833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494484835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742358_1534 (size=12301) 2024-11-13T10:40:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T10:40:24,859 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:24,859 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-13T10:40:24,861 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T10:40:24,862 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:24,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:24,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494484947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494484957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494484957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494484957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494484957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:24,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T10:40:25,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T10:40:25,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:25,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,089 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/1bb3999613b646969a8a1d45444cb36e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1bb3999613b646969a8a1d45444cb36e 2024-11-13T10:40:25,091 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/020060fdcc4c47c8ac59f4300e5a37bb as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/020060fdcc4c47c8ac59f4300e5a37bb 2024-11-13T10:40:25,096 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/A of de8553e5e3d4d26fa926e41bd56bc46e into 1bb3999613b646969a8a1d45444cb36e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:25,096 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/B of de8553e5e3d4d26fa926e41bd56bc46e into 020060fdcc4c47c8ac59f4300e5a37bb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:25,096 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:25,096 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/A, priority=13, startTime=1731494424599; duration=0sec 2024-11-13T10:40:25,096 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:25,097 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/B, priority=13, startTime=1731494424599; duration=0sec 2024-11-13T10:40:25,097 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:25,097 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:A 2024-11-13T10:40:25,097 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:25,097 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:25,097 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:B 2024-11-13T10:40:25,097 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:25,097 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/C is initiating minor compaction (all files) 2024-11-13T10:40:25,097 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/C in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,098 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/b04fa82d9e2f482ab46b1abfaac8bb48, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/356cb0de2ed54366bae445ee9d971d83, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/ee61b13e964b4ad7a117fa6b59ab9a84] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=36.1 K 2024-11-13T10:40:25,098 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04fa82d9e2f482ab46b1abfaac8bb48, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731494422626 2024-11-13T10:40:25,098 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 356cb0de2ed54366bae445ee9d971d83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731494422688 2024-11-13T10:40:25,098 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee61b13e964b4ad7a117fa6b59ab9a84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731494423375 2024-11-13T10:40:25,105 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#C#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:25,105 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/0fe954f9ce564829a2cee853b1aeb30f is 50, key is test_row_0/C:col10/1731494423375/Put/seqid=0 2024-11-13T10:40:25,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742359_1535 (size=12765) 2024-11-13T10:40:25,120 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/0fe954f9ce564829a2cee853b1aeb30f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0fe954f9ce564829a2cee853b1aeb30f 2024-11-13T10:40:25,125 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/C of de8553e5e3d4d26fa926e41bd56bc46e into 0fe954f9ce564829a2cee853b1aeb30f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:25,125 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:25,125 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/C, priority=13, startTime=1731494424599; duration=0sec 2024-11-13T10:40:25,125 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:25,125 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:C 2024-11-13T10:40:25,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494485168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494485168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494485168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494485169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:25,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494485169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:25,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T10:40:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:25,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:25,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T10:40:25,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/99cb0c5e9ee9429db7cc19ddc4cf55ff 2024-11-13T10:40:25,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/af04554397a544fd879b7b59485e0764 is 50, key is test_row_0/C:col10/1731494424149/Put/seqid=0 2024-11-13T10:40:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742360_1536 (size=12301) 2024-11-13T10:40:25,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T10:40:25,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:25,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494485477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T10:40:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494485477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494485477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:25,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494485478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:25,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494485478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:25,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T10:40:25,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:25,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T10:40:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:25,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/af04554397a544fd879b7b59485e0764 2024-11-13T10:40:25,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/a3ba9b25eafe47bf944b16d4785205e3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a3ba9b25eafe47bf944b16d4785205e3 2024-11-13T10:40:25,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a3ba9b25eafe47bf944b16d4785205e3, entries=200, sequenceid=281, filesize=14.4 K 2024-11-13T10:40:25,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/99cb0c5e9ee9429db7cc19ddc4cf55ff as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/99cb0c5e9ee9429db7cc19ddc4cf55ff 2024-11-13T10:40:25,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/99cb0c5e9ee9429db7cc19ddc4cf55ff, entries=150, sequenceid=281, filesize=12.0 K 2024-11-13T10:40:25,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/af04554397a544fd879b7b59485e0764 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/af04554397a544fd879b7b59485e0764 2024-11-13T10:40:25,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/af04554397a544fd879b7b59485e0764, entries=150, sequenceid=281, filesize=12.0 K 2024-11-13T10:40:25,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for de8553e5e3d4d26fa926e41bd56bc46e in 917ms, sequenceid=281, compaction requested=false 2024-11-13T10:40:25,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:25,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T10:40:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:25,799 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/d36ff84740d6467d9dec75cc1301be50 is 50, key is test_row_0/A:col10/1731494424828/Put/seqid=0 2024-11-13T10:40:25,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742361_1537 (size=12301) 2024-11-13T10:40:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T10:40:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:26,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:26,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494486057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494486058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494486059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494486069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494486070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494486176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494486177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494486177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494486180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494486187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,217 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/d36ff84740d6467d9dec75cc1301be50 2024-11-13T10:40:26,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a4c41dc39ba444e2a0639742f8f709e3 is 50, key is test_row_0/B:col10/1731494424828/Put/seqid=0 2024-11-13T10:40:26,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742362_1538 (size=12301) 2024-11-13T10:40:26,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494486387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494486387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494486387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494486397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494486407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,629 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a4c41dc39ba444e2a0639742f8f709e3 2024-11-13T10:40:26,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/6944d6ab1b7145b8bd89bb90dc9b5e96 is 50, key is test_row_0/C:col10/1731494424828/Put/seqid=0 2024-11-13T10:40:26,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742363_1539 (size=12301) 2024-11-13T10:40:26,668 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/6944d6ab1b7145b8bd89bb90dc9b5e96 2024-11-13T10:40:26,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/d36ff84740d6467d9dec75cc1301be50 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d36ff84740d6467d9dec75cc1301be50 2024-11-13T10:40:26,683 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d36ff84740d6467d9dec75cc1301be50, entries=150, sequenceid=295, filesize=12.0 K 2024-11-13T10:40:26,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a4c41dc39ba444e2a0639742f8f709e3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a4c41dc39ba444e2a0639742f8f709e3 2024-11-13T10:40:26,687 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a4c41dc39ba444e2a0639742f8f709e3, entries=150, sequenceid=295, filesize=12.0 K 2024-11-13T10:40:26,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/6944d6ab1b7145b8bd89bb90dc9b5e96 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/6944d6ab1b7145b8bd89bb90dc9b5e96 2024-11-13T10:40:26,692 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/6944d6ab1b7145b8bd89bb90dc9b5e96, entries=150, sequenceid=295, filesize=12.0 K 2024-11-13T10:40:26,692 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for de8553e5e3d4d26fa926e41bd56bc46e in 893ms, sequenceid=295, compaction requested=true 2024-11-13T10:40:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:26,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-13T10:40:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-13T10:40:26,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-13T10:40:26,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-11-13T10:40:26,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.8370 sec 2024-11-13T10:40:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:26,702 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:26,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T10:40:26,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/1428c947283044a7b64e6e6339e3a8d4 is 50, key is test_row_0/A:col10/1731494426068/Put/seqid=0 2024-11-13T10:40:26,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742364_1540 (size=14741) 2024-11-13T10:40:26,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/1428c947283044a7b64e6e6339e3a8d4 2024-11-13T10:40:26,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494486716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494486719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494486719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494486724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494486728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/0d3a2c111f3a424f86e504e518a019ac is 50, key is test_row_0/B:col10/1731494426068/Put/seqid=0 2024-11-13T10:40:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742365_1541 (size=12301) 2024-11-13T10:40:26,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/0d3a2c111f3a424f86e504e518a019ac 2024-11-13T10:40:26,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/90cb75fe20894eaf8aa29bda5caf913a is 50, key is test_row_0/C:col10/1731494426068/Put/seqid=0 2024-11-13T10:40:26,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742366_1542 (size=12301) 2024-11-13T10:40:26,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/90cb75fe20894eaf8aa29bda5caf913a 2024-11-13T10:40:26,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/1428c947283044a7b64e6e6339e3a8d4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1428c947283044a7b64e6e6339e3a8d4 2024-11-13T10:40:26,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494486827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494486839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494486848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:26,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1428c947283044a7b64e6e6339e3a8d4, entries=200, sequenceid=322, filesize=14.4 K 2024-11-13T10:40:26,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/0d3a2c111f3a424f86e504e518a019ac as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0d3a2c111f3a424f86e504e518a019ac 2024-11-13T10:40:26,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0d3a2c111f3a424f86e504e518a019ac, entries=150, sequenceid=322, filesize=12.0 K 2024-11-13T10:40:26,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/90cb75fe20894eaf8aa29bda5caf913a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/90cb75fe20894eaf8aa29bda5caf913a 2024-11-13T10:40:26,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/90cb75fe20894eaf8aa29bda5caf913a, entries=150, sequenceid=322, filesize=12.0 K 2024-11-13T10:40:26,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for de8553e5e3d4d26fa926e41bd56bc46e in 179ms, sequenceid=322, compaction requested=true 2024-11-13T10:40:26,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:26,882 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:26,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:26,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:26,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:26,882 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:26,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:26,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:26,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:26,884 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54548 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:26,884 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/A is initiating minor compaction (all files) 2024-11-13T10:40:26,884 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/A in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:26,884 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1bb3999613b646969a8a1d45444cb36e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a3ba9b25eafe47bf944b16d4785205e3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d36ff84740d6467d9dec75cc1301be50, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1428c947283044a7b64e6e6339e3a8d4] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=53.3 K 2024-11-13T10:40:26,885 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:26,885 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bb3999613b646969a8a1d45444cb36e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731494423375 2024-11-13T10:40:26,885 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/B is initiating minor compaction (all files) 2024-11-13T10:40:26,885 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/B in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:26,885 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/020060fdcc4c47c8ac59f4300e5a37bb, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/99cb0c5e9ee9429db7cc19ddc4cf55ff, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a4c41dc39ba444e2a0639742f8f709e3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0d3a2c111f3a424f86e504e518a019ac] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=48.5 K 2024-11-13T10:40:26,885 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3ba9b25eafe47bf944b16d4785205e3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731494424128 2024-11-13T10:40:26,885 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d36ff84740d6467d9dec75cc1301be50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731494424828 2024-11-13T10:40:26,885 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 020060fdcc4c47c8ac59f4300e5a37bb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731494423375 2024-11-13T10:40:26,886 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 99cb0c5e9ee9429db7cc19ddc4cf55ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731494424147 2024-11-13T10:40:26,886 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1428c947283044a7b64e6e6339e3a8d4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731494426057 2024-11-13T10:40:26,886 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting a4c41dc39ba444e2a0639742f8f709e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731494424828 2024-11-13T10:40:26,886 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d3a2c111f3a424f86e504e518a019ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731494426067 2024-11-13T10:40:26,896 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#A#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:26,897 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/9ef9f94f956844e49079ce1e904703e4 is 50, key is test_row_0/A:col10/1731494426068/Put/seqid=0 2024-11-13T10:40:26,897 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#B#compaction#463 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:26,898 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/2ceda50ea4e14396be8a8bdd0b57d38f is 50, key is test_row_0/B:col10/1731494426068/Put/seqid=0 2024-11-13T10:40:26,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742368_1544 (size=13051) 2024-11-13T10:40:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742367_1543 (size=13051) 2024-11-13T10:40:26,909 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/9ef9f94f956844e49079ce1e904703e4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/9ef9f94f956844e49079ce1e904703e4 2024-11-13T10:40:26,911 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/2ceda50ea4e14396be8a8bdd0b57d38f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/2ceda50ea4e14396be8a8bdd0b57d38f 2024-11-13T10:40:26,915 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/A of de8553e5e3d4d26fa926e41bd56bc46e into 9ef9f94f956844e49079ce1e904703e4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:26,915 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:26,915 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/A, priority=12, startTime=1731494426882; duration=0sec 2024-11-13T10:40:26,915 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:26,915 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:A 2024-11-13T10:40:26,915 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:26,920 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/B of de8553e5e3d4d26fa926e41bd56bc46e into 2ceda50ea4e14396be8a8bdd0b57d38f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:26,920 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:26,920 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/B, priority=12, startTime=1731494426882; duration=0sec 2024-11-13T10:40:26,920 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:26,920 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:B 2024-11-13T10:40:26,920 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:26,920 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): de8553e5e3d4d26fa926e41bd56bc46e/C is initiating minor compaction (all files) 2024-11-13T10:40:26,920 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of de8553e5e3d4d26fa926e41bd56bc46e/C in TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:26,920 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0fe954f9ce564829a2cee853b1aeb30f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/af04554397a544fd879b7b59485e0764, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/6944d6ab1b7145b8bd89bb90dc9b5e96, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/90cb75fe20894eaf8aa29bda5caf913a] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp, totalSize=48.5 K 2024-11-13T10:40:26,921 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fe954f9ce564829a2cee853b1aeb30f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731494423375 2024-11-13T10:40:26,927 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting af04554397a544fd879b7b59485e0764, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731494424147 2024-11-13T10:40:26,928 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6944d6ab1b7145b8bd89bb90dc9b5e96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731494424828 2024-11-13T10:40:26,928 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90cb75fe20894eaf8aa29bda5caf913a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731494426067 2024-11-13T10:40:26,938 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): de8553e5e3d4d26fa926e41bd56bc46e#C#compaction#464 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:26,939 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/e2850fe81f6d4b4ebef14fe196100613 is 50, key is test_row_0/C:col10/1731494426068/Put/seqid=0 2024-11-13T10:40:26,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742369_1545 (size=13051) 2024-11-13T10:40:26,972 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/e2850fe81f6d4b4ebef14fe196100613 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/e2850fe81f6d4b4ebef14fe196100613 2024-11-13T10:40:26,977 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in de8553e5e3d4d26fa926e41bd56bc46e/C of de8553e5e3d4d26fa926e41bd56bc46e into e2850fe81f6d4b4ebef14fe196100613(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:26,977 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:26,977 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., storeName=de8553e5e3d4d26fa926e41bd56bc46e/C, priority=12, startTime=1731494426882; duration=0sec 2024-11-13T10:40:26,977 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:26,977 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:C 2024-11-13T10:40:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T10:40:27,017 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:27,017 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-13T10:40:27,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T10:40:27,021 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:27,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:27,055 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:27,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:27,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ffb82a57617041e59fabcbffb6433711 is 50, key is test_row_0/A:col10/1731494427053/Put/seqid=0 2024-11-13T10:40:27,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742370_1546 (size=14741) 2024-11-13T10:40:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T10:40:27,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494487159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T10:40:27,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494487162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494487162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:27,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494487237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494487247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494487276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494487276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494487287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T10:40:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T10:40:27,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ffb82a57617041e59fabcbffb6433711 2024-11-13T10:40:27,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a41a0ded1bb545e4b3200ea1dfb4d8db is 50, key is test_row_0/B:col10/1731494427053/Put/seqid=0 2024-11-13T10:40:27,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T10:40:27,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:27,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494487497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494487497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742371_1547 (size=12301) 2024-11-13T10:40:27,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494487509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T10:40:27,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T10:40:27,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:27,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T10:40:27,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:27,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494487816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494487816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494487828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:27,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a41a0ded1bb545e4b3200ea1dfb4d8db 2024-11-13T10:40:27,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/6ee47d4e8a7c49b68f76081b243f5096 is 50, key is test_row_0/C:col10/1731494427053/Put/seqid=0 2024-11-13T10:40:27,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742372_1548 (size=12301) 2024-11-13T10:40:27,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/6ee47d4e8a7c49b68f76081b243f5096 2024-11-13T10:40:27,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T10:40:27,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:27,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:27,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:27,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/ffb82a57617041e59fabcbffb6433711 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ffb82a57617041e59fabcbffb6433711 2024-11-13T10:40:27,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ffb82a57617041e59fabcbffb6433711, entries=200, sequenceid=336, filesize=14.4 K 2024-11-13T10:40:27,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a41a0ded1bb545e4b3200ea1dfb4d8db as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a41a0ded1bb545e4b3200ea1dfb4d8db 2024-11-13T10:40:27,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a41a0ded1bb545e4b3200ea1dfb4d8db, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T10:40:27,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/6ee47d4e8a7c49b68f76081b243f5096 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/6ee47d4e8a7c49b68f76081b243f5096 2024-11-13T10:40:28,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/6ee47d4e8a7c49b68f76081b243f5096, entries=150, sequenceid=336, filesize=12.0 K 2024-11-13T10:40:28,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for de8553e5e3d4d26fa926e41bd56bc46e in 951ms, sequenceid=336, compaction requested=false 2024-11-13T10:40:28,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:28,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-13T10:40:28,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:28,120 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:40:28,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/fbc726e76b344de892e275c8bf0b4d26 is 50, key is test_row_0/A:col10/1731494427158/Put/seqid=0 2024-11-13T10:40:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T10:40:28,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742373_1549 (size=12301) 2024-11-13T10:40:28,159 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/fbc726e76b344de892e275c8bf0b4d26 2024-11-13T10:40:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/5c72bd353c9a46ad89073dc10b1bf65d is 50, key is test_row_0/B:col10/1731494427158/Put/seqid=0 2024-11-13T10:40:28,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742374_1550 (size=12301) 2024-11-13T10:40:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:28,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:28,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494488313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494488317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:28,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494488329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:28,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494488340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:28,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494488347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:28,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494488427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494488437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:28,613 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/5c72bd353c9a46ad89073dc10b1bf65d 2024-11-13T10:40:28,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494488646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:28,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/403dd0dd049c4fc8afad5722fe3d3734 is 50, key is test_row_0/C:col10/1731494427158/Put/seqid=0 2024-11-13T10:40:28,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494488649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:28,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742375_1551 (size=12301) 2024-11-13T10:40:28,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494488956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:28,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494488967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,083 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/403dd0dd049c4fc8afad5722fe3d3734 2024-11-13T10:40:29,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/fbc726e76b344de892e275c8bf0b4d26 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/fbc726e76b344de892e275c8bf0b4d26 2024-11-13T10:40:29,099 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/fbc726e76b344de892e275c8bf0b4d26, entries=150, sequenceid=362, filesize=12.0 K 2024-11-13T10:40:29,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/5c72bd353c9a46ad89073dc10b1bf65d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5c72bd353c9a46ad89073dc10b1bf65d 2024-11-13T10:40:29,112 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5c72bd353c9a46ad89073dc10b1bf65d, entries=150, sequenceid=362, filesize=12.0 K 2024-11-13T10:40:29,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/403dd0dd049c4fc8afad5722fe3d3734 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/403dd0dd049c4fc8afad5722fe3d3734 2024-11-13T10:40:29,129 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/403dd0dd049c4fc8afad5722fe3d3734, entries=150, sequenceid=362, filesize=12.0 K 2024-11-13T10:40:29,137 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for de8553e5e3d4d26fa926e41bd56bc46e in 1016ms, sequenceid=362, compaction requested=true 2024-11-13T10:40:29,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:29,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:29,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-13T10:40:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-13T10:40:29,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-13T10:40:29,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1170 sec 2024-11-13T10:40:29,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.1250 sec 2024-11-13T10:40:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T10:40:29,168 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:29,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-13T10:40:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T10:40:29,172 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:29,173 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:29,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T10:40:29,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-13T10:40:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:29,326 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T10:40:29,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/614a998b91c24fccbe40b18c1a7c3162 is 50, key is test_row_0/A:col10/1731494428274/Put/seqid=0 2024-11-13T10:40:29,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742376_1552 (size=12301) 2024-11-13T10:40:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:29,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:29,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494489451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494489452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494489469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T10:40:29,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494489477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494489477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494489578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494489579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494489587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,753 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/614a998b91c24fccbe40b18c1a7c3162 2024-11-13T10:40:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/50abb35cfbf34492bb6fd68a92a92a84 is 50, key is test_row_0/B:col10/1731494428274/Put/seqid=0 2024-11-13T10:40:29,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742377_1553 (size=12301) 2024-11-13T10:40:29,777 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/50abb35cfbf34492bb6fd68a92a92a84 2024-11-13T10:40:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/e3ec2d43b8f34759a9898dfa70daaf7a is 50, key is test_row_0/C:col10/1731494428274/Put/seqid=0 2024-11-13T10:40:29,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742378_1554 (size=12301) 2024-11-13T10:40:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T10:40:29,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494489800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494489801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:29,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494489801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494490117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494490117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494490119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,200 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/e3ec2d43b8f34759a9898dfa70daaf7a 2024-11-13T10:40:30,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/614a998b91c24fccbe40b18c1a7c3162 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/614a998b91c24fccbe40b18c1a7c3162 2024-11-13T10:40:30,215 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/614a998b91c24fccbe40b18c1a7c3162, entries=150, sequenceid=372, filesize=12.0 K 2024-11-13T10:40:30,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/50abb35cfbf34492bb6fd68a92a92a84 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/50abb35cfbf34492bb6fd68a92a92a84 2024-11-13T10:40:30,229 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/50abb35cfbf34492bb6fd68a92a92a84, entries=150, sequenceid=372, filesize=12.0 K 2024-11-13T10:40:30,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/e3ec2d43b8f34759a9898dfa70daaf7a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/e3ec2d43b8f34759a9898dfa70daaf7a 2024-11-13T10:40:30,235 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/e3ec2d43b8f34759a9898dfa70daaf7a, entries=150, sequenceid=372, filesize=12.0 K 2024-11-13T10:40:30,236 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for de8553e5e3d4d26fa926e41bd56bc46e in 910ms, sequenceid=372, compaction requested=true 2024-11-13T10:40:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-13T10:40:30,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-13T10:40:30,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-13T10:40:30,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0640 sec 2024-11-13T10:40:30,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.0710 sec 2024-11-13T10:40:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-13T10:40:30,307 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:30,308 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-13T10:40:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T10:40:30,309 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:30,310 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:30,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T10:40:30,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-13T10:40:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:30,464 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:40:30,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/16f7339f7bc749a08c294bc95169c5d1 is 50, key is test_row_0/A:col10/1731494429448/Put/seqid=0 2024-11-13T10:40:30,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742379_1555 (size=12301) 2024-11-13T10:40:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:30,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:30,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494490537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494490542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T10:40:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494490626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494490627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494490647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494490659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494490659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,876 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/16f7339f7bc749a08c294bc95169c5d1 2024-11-13T10:40:30,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494490879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494490879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:30,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:30,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/61a0b5e6dfcf496dacaff195b9e1d100 is 50, key is test_row_0/B:col10/1731494429448/Put/seqid=0 2024-11-13T10:40:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T10:40:30,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742380_1556 (size=12301) 2024-11-13T10:40:31,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494491197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:31,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:31,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494491197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:31,351 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/61a0b5e6dfcf496dacaff195b9e1d100 2024-11-13T10:40:31,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/eac4dce46fb74257bb1b765ae5a551ad is 50, key is test_row_0/C:col10/1731494429448/Put/seqid=0 2024-11-13T10:40:31,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742381_1557 (size=12301) 2024-11-13T10:40:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T10:40:31,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53074 deadline: 1731494491647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:31,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53056 deadline: 1731494491647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:31,657 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in de8553e5e3d4d26fa926e41bd56bc46e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:31,658 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in de8553e5e3d4d26fa926e41bd56bc46e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:31,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53084 deadline: 1731494491671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:31,673 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_2' in de8553e5e3d4d26fa926e41bd56bc46e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:31,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:31,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494491717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53036 deadline: 1731494491720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:31,777 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/eac4dce46fb74257bb1b765ae5a551ad 2024-11-13T10:40:31,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/16f7339f7bc749a08c294bc95169c5d1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/16f7339f7bc749a08c294bc95169c5d1 2024-11-13T10:40:31,809 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/16f7339f7bc749a08c294bc95169c5d1, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T10:40:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/61a0b5e6dfcf496dacaff195b9e1d100 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/61a0b5e6dfcf496dacaff195b9e1d100 2024-11-13T10:40:31,813 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/61a0b5e6dfcf496dacaff195b9e1d100, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T10:40:31,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/eac4dce46fb74257bb1b765ae5a551ad as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/eac4dce46fb74257bb1b765ae5a551ad 2024-11-13T10:40:31,818 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/eac4dce46fb74257bb1b765ae5a551ad, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T10:40:31,822 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for de8553e5e3d4d26fa926e41bd56bc46e in 1358ms, sequenceid=398, compaction requested=true 2024-11-13T10:40:31,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:31,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:31,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-13T10:40:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-13T10:40:31,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-13T10:40:31,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5150 sec 2024-11-13T10:40:31,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.5220 sec 2024-11-13T10:40:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T10:40:32,457 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:32,461 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-13T10:40:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T10:40:32,484 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:32,488 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:32,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:32,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T10:40:32,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-13T10:40:32,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:32,642 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T10:40:32,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/37387dfc09cb4f9fbee322ce7fefabb7 is 50, key is test_row_0/A:col10/1731494430540/Put/seqid=0 2024-11-13T10:40:32,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742382_1558 (size=12301) 2024-11-13T10:40:32,689 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/37387dfc09cb4f9fbee322ce7fefabb7 2024-11-13T10:40:32,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a4601a7c0363447bbbb90f0251c84b08 is 50, key is test_row_0/B:col10/1731494430540/Put/seqid=0 2024-11-13T10:40:32,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. as already flushing 2024-11-13T10:40:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:32,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742383_1559 (size=12301) 2024-11-13T10:40:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T10:40:32,934 INFO [Thread-2134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2134. 2024-11-13T10:40:32,934 DEBUG [Thread-2134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:32,934 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,935 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:32,935 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494492917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:32,937 INFO [Thread-2136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2136. 2024-11-13T10:40:32,937 DEBUG [Thread-2136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:32,937 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,937 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,937 INFO [Thread-2123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2123. 2024-11-13T10:40:32,937 DEBUG [Thread-2123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:32,937 DEBUG [Thread-2123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,937 DEBUG [Thread-2123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:32,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:32,941 INFO [Thread-2138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2138. 2024-11-13T10:40:32,941 DEBUG [Thread-2138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:32,941 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:32,941 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,953 INFO [Thread-2132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2132. 2024-11-13T10:40:32,953 DEBUG [Thread-2132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:32,953 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,953 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:32,954 INFO [Thread-2130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2130. 2024-11-13T10:40:32,954 DEBUG [Thread-2130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:32,954 DEBUG [Thread-2130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,954 DEBUG [Thread-2130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:32,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:33,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53040 deadline: 1731494493047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 , the old value is region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., hostname=770665a7984d,39101,1731494292956, seqNum=2 because the exception is null or not the one we care about 2024-11-13T10:40:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T10:40:33,155 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a4601a7c0363447bbbb90f0251c84b08 2024-11-13T10:40:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/76efd5b095a441999d9a755f4fe40767 is 50, key is test_row_0/C:col10/1731494430540/Put/seqid=0 2024-11-13T10:40:33,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742384_1560 (size=12301) 2024-11-13T10:40:33,171 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/76efd5b095a441999d9a755f4fe40767 2024-11-13T10:40:33,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/37387dfc09cb4f9fbee322ce7fefabb7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/37387dfc09cb4f9fbee322ce7fefabb7 2024-11-13T10:40:33,178 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/37387dfc09cb4f9fbee322ce7fefabb7, entries=150, sequenceid=408, filesize=12.0 K 2024-11-13T10:40:33,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/a4601a7c0363447bbbb90f0251c84b08 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a4601a7c0363447bbbb90f0251c84b08 2024-11-13T10:40:33,181 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a4601a7c0363447bbbb90f0251c84b08, entries=150, sequenceid=408, filesize=12.0 K 2024-11-13T10:40:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/76efd5b095a441999d9a755f4fe40767 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/76efd5b095a441999d9a755f4fe40767 2024-11-13T10:40:33,184 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/76efd5b095a441999d9a755f4fe40767, entries=150, sequenceid=408, filesize=12.0 K 2024-11-13T10:40:33,185 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for de8553e5e3d4d26fa926e41bd56bc46e in 543ms, sequenceid=408, compaction requested=true 2024-11-13T10:40:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-13T10:40:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-13T10:40:33,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-13T10:40:33,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 698 msec 2024-11-13T10:40:33,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 727 msec 2024-11-13T10:40:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:33,258 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:33,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T10:40:33,259 INFO [Thread-2125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2125. 2024-11-13T10:40:33,259 DEBUG [Thread-2125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:33,259 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:33,259 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:33,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:33,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/41dc3e21467d44bc9c51d52540d6d85d is 50, key is test_row_0/A:col10/1731494432931/Put/seqid=0 2024-11-13T10:40:33,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742385_1561 (size=12301) 2024-11-13T10:40:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-13T10:40:33,617 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:33,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/41dc3e21467d44bc9c51d52540d6d85d 2024-11-13T10:40:33,668 INFO [Thread-2119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2119. 2024-11-13T10:40:33,669 DEBUG [Thread-2119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:33,669 DEBUG [Thread-2119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:33,669 DEBUG [Thread-2119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:33,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:33,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/0c2ce1f88a6840e7ac34154503050de7 is 50, key is test_row_0/B:col10/1731494432931/Put/seqid=0 2024-11-13T10:40:33,678 INFO [Thread-2121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2121. 2024-11-13T10:40:33,679 DEBUG [Thread-2121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:33,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742386_1562 (size=12301) 2024-11-13T10:40:33,679 DEBUG [Thread-2121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:33,679 DEBUG [Thread-2121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:33,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:33,680 INFO [Thread-2127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2127. 2024-11-13T10:40:33,680 DEBUG [Thread-2127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:40:33,680 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:33,680 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:33,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1077 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3231 rows 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1077 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3225 rows 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1080 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3240 rows 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1081 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3243 rows 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1068 2024-11-13T10:40:33,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3204 rows 2024-11-13T10:40:33,680 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T10:40:33,681 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T10:40:33,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:33,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:33,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:40:33,685 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T10:40:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T10:40:33,689 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494433689"}]},"ts":"1731494433689"} 2024-11-13T10:40:33,690 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T10:40:33,690 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T10:40:33,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T10:40:33,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de8553e5e3d4d26fa926e41bd56bc46e, UNASSIGN}] 2024-11-13T10:40:33,694 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de8553e5e3d4d26fa926e41bd56bc46e, UNASSIGN 2024-11-13T10:40:33,695 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=de8553e5e3d4d26fa926e41bd56bc46e, regionState=CLOSING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:40:33,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=de8553e5e3d4d26fa926e41bd56bc46e, UNASSIGN because future has completed 2024-11-13T10:40:33,696 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T10:40:33,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:40:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T10:40:33,849 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:33,849 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T10:40:33,849 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing de8553e5e3d4d26fa926e41bd56bc46e, disabling compactions & flushes 2024-11-13T10:40:33,849 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T10:40:34,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/0c2ce1f88a6840e7ac34154503050de7 2024-11-13T10:40:34,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/f5270925d433462bb21d8304f693a464 is 50, key is test_row_0/C:col10/1731494432931/Put/seqid=0 2024-11-13T10:40:34,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742387_1563 (size=12301) 2024-11-13T10:40:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T10:40:34,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/f5270925d433462bb21d8304f693a464 2024-11-13T10:40:34,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/41dc3e21467d44bc9c51d52540d6d85d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/41dc3e21467d44bc9c51d52540d6d85d 2024-11-13T10:40:34,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/41dc3e21467d44bc9c51d52540d6d85d, entries=150, sequenceid=435, filesize=12.0 K 2024-11-13T10:40:34,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/0c2ce1f88a6840e7ac34154503050de7 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0c2ce1f88a6840e7ac34154503050de7 2024-11-13T10:40:34,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0c2ce1f88a6840e7ac34154503050de7, entries=150, sequenceid=435, filesize=12.0 K 2024-11-13T10:40:34,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/f5270925d433462bb21d8304f693a464 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/f5270925d433462bb21d8304f693a464 2024-11-13T10:40:34,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/f5270925d433462bb21d8304f693a464, entries=150, sequenceid=435, filesize=12.0 K 2024-11-13T10:40:34,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=20.13 KB/20610 for de8553e5e3d4d26fa926e41bd56bc46e in 1251ms, sequenceid=435, compaction requested=true 2024-11-13T10:40:34,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for de8553e5e3d4d26fa926e41bd56bc46e: 2024-11-13T10:40:34,509 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:34,509 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:34,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:34,509 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. after waiting 0 ms 2024-11-13T10:40:34,509 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:34,509 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. because compaction request was cancelled 2024-11-13T10:40:34,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:34,509 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:A 2024-11-13T10:40:34,509 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. because compaction request was cancelled 2024-11-13T10:40:34,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:B, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:34,509 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:B 2024-11-13T10:40:34,509 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing de8553e5e3d4d26fa926e41bd56bc46e 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-13T10:40:34,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:34,509 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. because compaction request was cancelled 2024-11-13T10:40:34,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store de8553e5e3d4d26fa926e41bd56bc46e:C, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:34,509 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: de8553e5e3d4d26fa926e41bd56bc46e:C 2024-11-13T10:40:34,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:34,512 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/1378c9ca5fa04a83b7ad07dd6a10e123 is 50, key is test_row_0/A:col10/1731494433667/Put/seqid=0 2024-11-13T10:40:34,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742388_1564 (size=12301) 2024-11-13T10:40:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T10:40:34,915 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/1378c9ca5fa04a83b7ad07dd6a10e123 2024-11-13T10:40:34,934 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/eb5a63fae61f4381bd33fed6a939f79a is 50, key is test_row_0/B:col10/1731494433667/Put/seqid=0 2024-11-13T10:40:34,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742389_1565 (size=12301) 2024-11-13T10:40:35,338 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/eb5a63fae61f4381bd33fed6a939f79a 2024-11-13T10:40:35,343 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/df6f3045b8574dd4a24c3a5ce2cc014a is 50, key is test_row_0/C:col10/1731494433667/Put/seqid=0 2024-11-13T10:40:35,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742390_1566 (size=12301) 2024-11-13T10:40:35,747 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/df6f3045b8574dd4a24c3a5ce2cc014a 2024-11-13T10:40:35,751 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/A/1378c9ca5fa04a83b7ad07dd6a10e123 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1378c9ca5fa04a83b7ad07dd6a10e123 2024-11-13T10:40:35,755 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1378c9ca5fa04a83b7ad07dd6a10e123, entries=150, sequenceid=441, filesize=12.0 K 2024-11-13T10:40:35,758 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/B/eb5a63fae61f4381bd33fed6a939f79a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/eb5a63fae61f4381bd33fed6a939f79a 2024-11-13T10:40:35,763 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/eb5a63fae61f4381bd33fed6a939f79a, entries=150, sequenceid=441, filesize=12.0 K 2024-11-13T10:40:35,765 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/.tmp/C/df6f3045b8574dd4a24c3a5ce2cc014a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/df6f3045b8574dd4a24c3a5ce2cc014a 2024-11-13T10:40:35,768 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/df6f3045b8574dd4a24c3a5ce2cc014a, entries=150, sequenceid=441, filesize=12.0 K 2024-11-13T10:40:35,770 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for de8553e5e3d4d26fa926e41bd56bc46e in 1260ms, sequenceid=441, compaction requested=true 2024-11-13T10:40:35,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/80f7a03b4c804231b0e4fb783d561f74, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d91ec386b0cd44c3a71dba53ba3e0581, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ed6d149d6cce4af2a991fafe7a1bac3b, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/23377cd07ce542a5a0d45d3dcacf42f5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/19c55a17584b491eb66dd930db52bdc9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/c289fccdf08a49ac84b070bcb3527018, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/fddffcadf2ee4174b34fe1ce47de7ac3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/8f4660948ced4736b641cc8dcfb247e9, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d8e616d148d5445590db49f2c2ca1072, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/472f045f48334520853e0fee1bcd038d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ac502e5c413640f6aa6baec10847e505, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/4b65f3529c6c4cbda1614465b06b366f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ca1300a7f99947119d14a79b725280e0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/38b5a951f73d4a078c27cf7bae76f437, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/0cab8a0b06c64ef88f9dc1dc650fe250, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/b00cc834fd2d4c48bd8c9165e9175f0c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a53774e7c14f4b59a4dba841d8c2556c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a510552f0d7a467cbfb8bf5eed32a457, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1bb3999613b646969a8a1d45444cb36e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a3ba9b25eafe47bf944b16d4785205e3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d36ff84740d6467d9dec75cc1301be50, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1428c947283044a7b64e6e6339e3a8d4] to archive 2024-11-13T10:40:35,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:40:35,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/80f7a03b4c804231b0e4fb783d561f74 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/80f7a03b4c804231b0e4fb783d561f74 2024-11-13T10:40:35,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d91ec386b0cd44c3a71dba53ba3e0581 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d91ec386b0cd44c3a71dba53ba3e0581 2024-11-13T10:40:35,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ed6d149d6cce4af2a991fafe7a1bac3b to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ed6d149d6cce4af2a991fafe7a1bac3b 2024-11-13T10:40:35,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/23377cd07ce542a5a0d45d3dcacf42f5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/23377cd07ce542a5a0d45d3dcacf42f5 2024-11-13T10:40:35,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/19c55a17584b491eb66dd930db52bdc9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/19c55a17584b491eb66dd930db52bdc9 2024-11-13T10:40:35,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/c289fccdf08a49ac84b070bcb3527018 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/c289fccdf08a49ac84b070bcb3527018 2024-11-13T10:40:35,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/fddffcadf2ee4174b34fe1ce47de7ac3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/fddffcadf2ee4174b34fe1ce47de7ac3 2024-11-13T10:40:35,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/8f4660948ced4736b641cc8dcfb247e9 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/8f4660948ced4736b641cc8dcfb247e9 2024-11-13T10:40:35,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d8e616d148d5445590db49f2c2ca1072 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d8e616d148d5445590db49f2c2ca1072 2024-11-13T10:40:35,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/472f045f48334520853e0fee1bcd038d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/472f045f48334520853e0fee1bcd038d 2024-11-13T10:40:35,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ac502e5c413640f6aa6baec10847e505 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ac502e5c413640f6aa6baec10847e505 2024-11-13T10:40:35,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/4b65f3529c6c4cbda1614465b06b366f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/4b65f3529c6c4cbda1614465b06b366f 2024-11-13T10:40:35,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ca1300a7f99947119d14a79b725280e0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ca1300a7f99947119d14a79b725280e0 2024-11-13T10:40:35,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/38b5a951f73d4a078c27cf7bae76f437 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/38b5a951f73d4a078c27cf7bae76f437 2024-11-13T10:40:35,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/0cab8a0b06c64ef88f9dc1dc650fe250 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/0cab8a0b06c64ef88f9dc1dc650fe250 2024-11-13T10:40:35,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/b00cc834fd2d4c48bd8c9165e9175f0c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/b00cc834fd2d4c48bd8c9165e9175f0c 2024-11-13T10:40:35,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a53774e7c14f4b59a4dba841d8c2556c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a53774e7c14f4b59a4dba841d8c2556c 2024-11-13T10:40:35,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a510552f0d7a467cbfb8bf5eed32a457 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a510552f0d7a467cbfb8bf5eed32a457 2024-11-13T10:40:35,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1bb3999613b646969a8a1d45444cb36e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1bb3999613b646969a8a1d45444cb36e 2024-11-13T10:40:35,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a3ba9b25eafe47bf944b16d4785205e3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/a3ba9b25eafe47bf944b16d4785205e3 2024-11-13T10:40:35,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d36ff84740d6467d9dec75cc1301be50 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/d36ff84740d6467d9dec75cc1301be50 2024-11-13T10:40:35,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1428c947283044a7b64e6e6339e3a8d4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1428c947283044a7b64e6e6339e3a8d4 2024-11-13T10:40:35,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/52bdf49059f5448484a53b016597dfb1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/c2b34c46362c4a19a722817fe19dad8a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0c252082363c47d98ad6a03acf75453c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/4ee8df0413a94c1a877333ae8d69ffba, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7c78d7b765e44b928e02c5e25e5eda16, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/9dfe15733efe4ac9b785c159a6051030, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/33eb15fb7b0a4035bf0f7759b35aaed7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5bd7107a4e2a4c4e9a31ff7dc556d542, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/293c53eecdf242ac99691ec736d520f2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/34946a51ed044f91b1a2069beac725a7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/6deb60734e83467f96c27eb9fbf3cf37, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7503495d41f84e4c8145ab3ee52b2d50, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5ebbe62845144786a3cccf6939a77fde, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/b1719abf1881419e809fcfab330694f5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a5ff9e9ab08447dbb8989f976a87bdcc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1df8be75f856474187e1f35331b3fb99, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1cb2aed3519849eaadacccd91888c041, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/020060fdcc4c47c8ac59f4300e5a37bb, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/ca88faed8feb4296aa5d0787b67032a7, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/99cb0c5e9ee9429db7cc19ddc4cf55ff, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a4c41dc39ba444e2a0639742f8f709e3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0d3a2c111f3a424f86e504e518a019ac] to archive 2024-11-13T10:40:35,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:40:35,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/52bdf49059f5448484a53b016597dfb1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/52bdf49059f5448484a53b016597dfb1 2024-11-13T10:40:35,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/c2b34c46362c4a19a722817fe19dad8a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/c2b34c46362c4a19a722817fe19dad8a 2024-11-13T10:40:35,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0c252082363c47d98ad6a03acf75453c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0c252082363c47d98ad6a03acf75453c 2024-11-13T10:40:35,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/4ee8df0413a94c1a877333ae8d69ffba to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/4ee8df0413a94c1a877333ae8d69ffba 2024-11-13T10:40:35,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7c78d7b765e44b928e02c5e25e5eda16 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7c78d7b765e44b928e02c5e25e5eda16 2024-11-13T10:40:35,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/9dfe15733efe4ac9b785c159a6051030 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/9dfe15733efe4ac9b785c159a6051030 2024-11-13T10:40:35,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/33eb15fb7b0a4035bf0f7759b35aaed7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/33eb15fb7b0a4035bf0f7759b35aaed7 2024-11-13T10:40:35,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5bd7107a4e2a4c4e9a31ff7dc556d542 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5bd7107a4e2a4c4e9a31ff7dc556d542 2024-11-13T10:40:35,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/293c53eecdf242ac99691ec736d520f2 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/293c53eecdf242ac99691ec736d520f2 2024-11-13T10:40:35,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/34946a51ed044f91b1a2069beac725a7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/34946a51ed044f91b1a2069beac725a7 2024-11-13T10:40:35,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/6deb60734e83467f96c27eb9fbf3cf37 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/6deb60734e83467f96c27eb9fbf3cf37 2024-11-13T10:40:35,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7503495d41f84e4c8145ab3ee52b2d50 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/7503495d41f84e4c8145ab3ee52b2d50 2024-11-13T10:40:35,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5ebbe62845144786a3cccf6939a77fde to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5ebbe62845144786a3cccf6939a77fde 2024-11-13T10:40:35,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/b1719abf1881419e809fcfab330694f5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/b1719abf1881419e809fcfab330694f5 2024-11-13T10:40:35,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a5ff9e9ab08447dbb8989f976a87bdcc to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a5ff9e9ab08447dbb8989f976a87bdcc 2024-11-13T10:40:35,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1df8be75f856474187e1f35331b3fb99 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1df8be75f856474187e1f35331b3fb99 2024-11-13T10:40:35,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1cb2aed3519849eaadacccd91888c041 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/1cb2aed3519849eaadacccd91888c041 2024-11-13T10:40:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T10:40:35,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/020060fdcc4c47c8ac59f4300e5a37bb to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/020060fdcc4c47c8ac59f4300e5a37bb 2024-11-13T10:40:35,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/ca88faed8feb4296aa5d0787b67032a7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/ca88faed8feb4296aa5d0787b67032a7 2024-11-13T10:40:35,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/99cb0c5e9ee9429db7cc19ddc4cf55ff to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/99cb0c5e9ee9429db7cc19ddc4cf55ff 2024-11-13T10:40:35,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a4c41dc39ba444e2a0639742f8f709e3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a4c41dc39ba444e2a0639742f8f709e3 2024-11-13T10:40:35,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0d3a2c111f3a424f86e504e518a019ac to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0d3a2c111f3a424f86e504e518a019ac 2024-11-13T10:40:35,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4b5d000b3c8a40528824d47a321ff36f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/7be44fa23a834f40b817a489d1f103a5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d92c40906e3a4078be70a36ca9acba7c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/a7f1344c2b6c47ce99734f78b303332f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/5c3dffc444cd481eac9d34951919d0ba, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/9ae8791f3b1b4142959209eff7dba046, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/420912721cb743d898c9647fa4e74d7e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4214ad9d77a9442fa1b1041ae6c6aaaa, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/84a6126811754792a8d38b5dc04532be, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/113f70a26cb74b6f8bb96269a1cbcada, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0bde1417b0644f93a8de36095793ca62, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0ffc4a2442c34afe9845b519319ac897, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d669e25f59bf44a3b01f003dd2c9e136, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/993e5d95d4234eb6b7d1e6a910da0d63, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/b04fa82d9e2f482ab46b1abfaac8bb48, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/afd308abba5249b88bf1b4ed2a28c829, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/356cb0de2ed54366bae445ee9d971d83, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0fe954f9ce564829a2cee853b1aeb30f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/ee61b13e964b4ad7a117fa6b59ab9a84, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/af04554397a544fd879b7b59485e0764, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/6944d6ab1b7145b8bd89bb90dc9b5e96, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/90cb75fe20894eaf8aa29bda5caf913a] to archive 2024-11-13T10:40:35,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:40:35,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4b5d000b3c8a40528824d47a321ff36f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4b5d000b3c8a40528824d47a321ff36f 2024-11-13T10:40:35,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/7be44fa23a834f40b817a489d1f103a5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/7be44fa23a834f40b817a489d1f103a5 2024-11-13T10:40:35,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d92c40906e3a4078be70a36ca9acba7c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d92c40906e3a4078be70a36ca9acba7c 2024-11-13T10:40:35,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/a7f1344c2b6c47ce99734f78b303332f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/a7f1344c2b6c47ce99734f78b303332f 2024-11-13T10:40:35,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/5c3dffc444cd481eac9d34951919d0ba to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/5c3dffc444cd481eac9d34951919d0ba 2024-11-13T10:40:35,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/9ae8791f3b1b4142959209eff7dba046 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/9ae8791f3b1b4142959209eff7dba046 2024-11-13T10:40:35,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/420912721cb743d898c9647fa4e74d7e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/420912721cb743d898c9647fa4e74d7e 2024-11-13T10:40:35,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4214ad9d77a9442fa1b1041ae6c6aaaa to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/4214ad9d77a9442fa1b1041ae6c6aaaa 2024-11-13T10:40:35,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/84a6126811754792a8d38b5dc04532be to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/84a6126811754792a8d38b5dc04532be 2024-11-13T10:40:35,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/113f70a26cb74b6f8bb96269a1cbcada to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/113f70a26cb74b6f8bb96269a1cbcada 2024-11-13T10:40:35,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0bde1417b0644f93a8de36095793ca62 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0bde1417b0644f93a8de36095793ca62 2024-11-13T10:40:35,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0ffc4a2442c34afe9845b519319ac897 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0ffc4a2442c34afe9845b519319ac897 2024-11-13T10:40:35,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d669e25f59bf44a3b01f003dd2c9e136 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/d669e25f59bf44a3b01f003dd2c9e136 2024-11-13T10:40:35,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/993e5d95d4234eb6b7d1e6a910da0d63 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/993e5d95d4234eb6b7d1e6a910da0d63 2024-11-13T10:40:35,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/b04fa82d9e2f482ab46b1abfaac8bb48 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/b04fa82d9e2f482ab46b1abfaac8bb48 2024-11-13T10:40:35,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/afd308abba5249b88bf1b4ed2a28c829 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/afd308abba5249b88bf1b4ed2a28c829 2024-11-13T10:40:35,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/356cb0de2ed54366bae445ee9d971d83 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/356cb0de2ed54366bae445ee9d971d83 2024-11-13T10:40:35,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0fe954f9ce564829a2cee853b1aeb30f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/0fe954f9ce564829a2cee853b1aeb30f 2024-11-13T10:40:35,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/ee61b13e964b4ad7a117fa6b59ab9a84 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/ee61b13e964b4ad7a117fa6b59ab9a84 2024-11-13T10:40:35,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/af04554397a544fd879b7b59485e0764 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/af04554397a544fd879b7b59485e0764 2024-11-13T10:40:35,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/6944d6ab1b7145b8bd89bb90dc9b5e96 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/6944d6ab1b7145b8bd89bb90dc9b5e96 2024-11-13T10:40:35,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/90cb75fe20894eaf8aa29bda5caf913a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/90cb75fe20894eaf8aa29bda5caf913a 2024-11-13T10:40:35,956 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/recovered.edits/444.seqid, newMaxSeqId=444, maxSeqId=1 2024-11-13T10:40:35,956 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. 2024-11-13T10:40:35,957 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for de8553e5e3d4d26fa926e41bd56bc46e: Waiting for close lock at 1731494433849Running coprocessor pre-close hooks at 1731494433849Disabling compacts and flushes for region at 1731494433849Disabling writes for close at 1731494434509 (+660 ms)Obtaining lock to block concurrent updates at 1731494434509Preparing flush snapshotting stores in de8553e5e3d4d26fa926e41bd56bc46e at 1731494434509Finished memstore snapshotting TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731494434509Flushing stores of TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e. at 1731494434510 (+1 ms)Flushing de8553e5e3d4d26fa926e41bd56bc46e/A: creating writer at 1731494434510Flushing de8553e5e3d4d26fa926e41bd56bc46e/A: appending metadata at 1731494434511 (+1 ms)Flushing de8553e5e3d4d26fa926e41bd56bc46e/A: closing flushed file at 1731494434511Flushing de8553e5e3d4d26fa926e41bd56bc46e/B: creating writer at 1731494434931 (+420 ms)Flushing de8553e5e3d4d26fa926e41bd56bc46e/B: appending metadata at 1731494434934 (+3 ms)Flushing de8553e5e3d4d26fa926e41bd56bc46e/B: closing flushed file at 1731494434934Flushing de8553e5e3d4d26fa926e41bd56bc46e/C: creating writer at 1731494435341 (+407 ms)Flushing de8553e5e3d4d26fa926e41bd56bc46e/C: appending metadata at 1731494435343 (+2 ms)Flushing de8553e5e3d4d26fa926e41bd56bc46e/C: closing flushed file at 1731494435343Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f64aa59: reopening flushed file at 1731494435750 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33addab2: reopening flushed file at 1731494435755 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d18d877: reopening flushed file at 1731494435763 (+8 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for de8553e5e3d4d26fa926e41bd56bc46e in 1260ms, sequenceid=441, compaction requested=true at 1731494435770 (+7 ms)Writing region close event to WAL at 1731494435953 (+183 ms)Running coprocessor post-close hooks at 1731494435956 (+3 ms)Closed at 1731494435956 2024-11-13T10:40:35,958 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:35,959 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=de8553e5e3d4d26fa926e41bd56bc46e, regionState=CLOSED 2024-11-13T10:40:35,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:40:35,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-13T10:40:35,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure de8553e5e3d4d26fa926e41bd56bc46e, server=770665a7984d,39101,1731494292956 in 2.2650 sec 2024-11-13T10:40:35,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-13T10:40:35,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=de8553e5e3d4d26fa926e41bd56bc46e, UNASSIGN in 2.2690 sec 2024-11-13T10:40:35,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-13T10:40:35,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2730 sec 2024-11-13T10:40:35,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494435966"}]},"ts":"1731494435966"} 2024-11-13T10:40:35,967 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T10:40:35,967 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T10:40:35,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2830 sec 2024-11-13T10:40:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T10:40:37,847 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:37,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T10:40:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:37,850 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-13T10:40:37,851 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:37,855 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:37,857 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/recovered.edits] 2024-11-13T10:40:37,860 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1378c9ca5fa04a83b7ad07dd6a10e123 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/1378c9ca5fa04a83b7ad07dd6a10e123 2024-11-13T10:40:37,861 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/16f7339f7bc749a08c294bc95169c5d1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/16f7339f7bc749a08c294bc95169c5d1 2024-11-13T10:40:37,863 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/37387dfc09cb4f9fbee322ce7fefabb7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/37387dfc09cb4f9fbee322ce7fefabb7 2024-11-13T10:40:37,864 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/41dc3e21467d44bc9c51d52540d6d85d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/41dc3e21467d44bc9c51d52540d6d85d 2024-11-13T10:40:37,865 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/614a998b91c24fccbe40b18c1a7c3162 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/614a998b91c24fccbe40b18c1a7c3162 2024-11-13T10:40:37,866 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/9ef9f94f956844e49079ce1e904703e4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/9ef9f94f956844e49079ce1e904703e4 2024-11-13T10:40:37,867 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/fbc726e76b344de892e275c8bf0b4d26 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/fbc726e76b344de892e275c8bf0b4d26 2024-11-13T10:40:37,868 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ffb82a57617041e59fabcbffb6433711 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/A/ffb82a57617041e59fabcbffb6433711 2024-11-13T10:40:37,876 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0c2ce1f88a6840e7ac34154503050de7 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/0c2ce1f88a6840e7ac34154503050de7 2024-11-13T10:40:37,877 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/2ceda50ea4e14396be8a8bdd0b57d38f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/2ceda50ea4e14396be8a8bdd0b57d38f 2024-11-13T10:40:37,878 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/50abb35cfbf34492bb6fd68a92a92a84 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/50abb35cfbf34492bb6fd68a92a92a84 2024-11-13T10:40:37,879 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5c72bd353c9a46ad89073dc10b1bf65d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/5c72bd353c9a46ad89073dc10b1bf65d 2024-11-13T10:40:37,880 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/61a0b5e6dfcf496dacaff195b9e1d100 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/61a0b5e6dfcf496dacaff195b9e1d100 2024-11-13T10:40:37,881 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a41a0ded1bb545e4b3200ea1dfb4d8db to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a41a0ded1bb545e4b3200ea1dfb4d8db 2024-11-13T10:40:37,882 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a4601a7c0363447bbbb90f0251c84b08 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/a4601a7c0363447bbbb90f0251c84b08 2024-11-13T10:40:37,883 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/eb5a63fae61f4381bd33fed6a939f79a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/B/eb5a63fae61f4381bd33fed6a939f79a 2024-11-13T10:40:37,884 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/403dd0dd049c4fc8afad5722fe3d3734 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/403dd0dd049c4fc8afad5722fe3d3734 2024-11-13T10:40:37,885 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/6ee47d4e8a7c49b68f76081b243f5096 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/6ee47d4e8a7c49b68f76081b243f5096 2024-11-13T10:40:37,886 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/76efd5b095a441999d9a755f4fe40767 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/76efd5b095a441999d9a755f4fe40767 2024-11-13T10:40:37,887 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/df6f3045b8574dd4a24c3a5ce2cc014a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/df6f3045b8574dd4a24c3a5ce2cc014a 2024-11-13T10:40:37,888 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/e2850fe81f6d4b4ebef14fe196100613 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/e2850fe81f6d4b4ebef14fe196100613 2024-11-13T10:40:37,889 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/e3ec2d43b8f34759a9898dfa70daaf7a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/e3ec2d43b8f34759a9898dfa70daaf7a 2024-11-13T10:40:37,890 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/eac4dce46fb74257bb1b765ae5a551ad to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/eac4dce46fb74257bb1b765ae5a551ad 2024-11-13T10:40:37,891 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/f5270925d433462bb21d8304f693a464 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/C/f5270925d433462bb21d8304f693a464 2024-11-13T10:40:37,893 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/recovered.edits/444.seqid to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e/recovered.edits/444.seqid 2024-11-13T10:40:37,893 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/de8553e5e3d4d26fa926e41bd56bc46e 2024-11-13T10:40:37,893 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T10:40:37,896 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:37,898 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T10:40:37,900 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T10:40:37,901 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:37,901 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T10:40:37,901 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731494437901"}]},"ts":"9223372036854775807"} 2024-11-13T10:40:37,903 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T10:40:37,903 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => de8553e5e3d4d26fa926e41bd56bc46e, NAME => 'TestAcidGuarantees,,1731494412222.de8553e5e3d4d26fa926e41bd56bc46e.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T10:40:37,903 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T10:40:37,904 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731494437903"}]},"ts":"9223372036854775807"} 2024-11-13T10:40:37,905 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T10:40:37,906 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:37,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-11-13T10:40:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-13T10:40:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T10:40:37,961 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:37,974 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 241), OpenFileDescriptor=445 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=768 (was 766) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=516 (was 597) 2024-11-13T10:40:37,990 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=768, ProcessCount=11, AvailableMemoryMB=514 2024-11-13T10:40:37,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T10:40:37,993 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T10:40:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:37,996 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T10:40:37,996 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:37,997 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T10:40:37,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-13T10:40:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T10:40:38,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742391_1567 (size=959) 2024-11-13T10:40:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T10:40:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T10:40:38,414 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617 2024-11-13T10:40:38,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742392_1568 (size=53) 2024-11-13T10:40:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T10:40:38,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:40:38,909 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c32615157847c6ad00128cef38e49033, disabling compactions & flushes 2024-11-13T10:40:38,909 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:38,909 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:38,909 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. after waiting 0 ms 2024-11-13T10:40:38,909 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:38,909 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:38,909 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c32615157847c6ad00128cef38e49033: Waiting for close lock at 1731494438909Disabling compacts and flushes for region at 1731494438909Disabling writes for close at 1731494438909Writing region close event to WAL at 1731494438909Closed at 1731494438909 2024-11-13T10:40:38,910 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T10:40:38,910 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731494438910"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731494438910"}]},"ts":"1731494438910"} 2024-11-13T10:40:38,912 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T10:40:38,912 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T10:40:38,912 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494438912"}]},"ts":"1731494438912"} 2024-11-13T10:40:38,914 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T10:40:38,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, ASSIGN}] 2024-11-13T10:40:38,915 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, ASSIGN 2024-11-13T10:40:38,916 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, ASSIGN; state=OFFLINE, location=770665a7984d,39101,1731494292956; forceNewPlan=false, retain=false 2024-11-13T10:40:39,067 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c32615157847c6ad00128cef38e49033, regionState=OPENING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:40:39,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, ASSIGN because future has completed 2024-11-13T10:40:39,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:40:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T10:40:39,223 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:39,224 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:40:39,224 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,224 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:40:39,224 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,224 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,225 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,227 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32615157847c6ad00128cef38e49033 columnFamilyName A 2024-11-13T10:40:39,227 DEBUG [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:39,227 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(327): Store=c32615157847c6ad00128cef38e49033/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:40:39,227 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,228 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32615157847c6ad00128cef38e49033 columnFamilyName B 2024-11-13T10:40:39,228 DEBUG [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:39,229 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(327): Store=c32615157847c6ad00128cef38e49033/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:40:39,229 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,230 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32615157847c6ad00128cef38e49033 columnFamilyName C 2024-11-13T10:40:39,230 DEBUG [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:39,231 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(327): Store=c32615157847c6ad00128cef38e49033/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:40:39,231 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,231 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,232 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,232 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,232 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,233 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:40:39,234 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,235 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T10:40:39,236 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened c32615157847c6ad00128cef38e49033; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68620262, jitterRate=0.02252158522605896}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:40:39,236 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:39,236 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for c32615157847c6ad00128cef38e49033: Running coprocessor pre-open hook at 1731494439224Writing region info on filesystem at 1731494439224Initializing all the Stores at 1731494439225 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494439225Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494439225Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494439225Cleaning up temporary data from old regions at 1731494439232 (+7 ms)Running coprocessor post-open hooks at 1731494439236 (+4 ms)Region opened successfully at 1731494439236 2024-11-13T10:40:39,237 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., pid=148, masterSystemTime=1731494439220 2024-11-13T10:40:39,239 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:39,239 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:39,240 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c32615157847c6ad00128cef38e49033, regionState=OPEN, openSeqNum=2, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:40:39,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:40:39,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-13T10:40:39,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 in 174 msec 2024-11-13T10:40:39,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-13T10:40:39,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, ASSIGN in 329 msec 2024-11-13T10:40:39,246 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T10:40:39,246 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494439246"}]},"ts":"1731494439246"} 2024-11-13T10:40:39,247 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T10:40:39,248 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T10:40:39,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2550 sec 2024-11-13T10:40:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T10:40:40,147 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:40,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cec06c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:40,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:40,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:40,150 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ebc8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:40,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:40,152 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bdfd2bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:40,153 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:40,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:40,157 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T10:40:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 770665a7984d,44523,1731494292131 2024-11-13T10:40:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58f42fb6 2024-11-13T10:40:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T10:40:40,160 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T10:40:40,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T10:40:40,163 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T10:40:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T10:40:40,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742393_1569 (size=995) 2024-11-13T10:40:40,591 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T10:40:40,591 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T10:40:40,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T10:40:40,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, REOPEN/MOVE}] 2024-11-13T10:40:40,596 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, REOPEN/MOVE 2024-11-13T10:40:40,597 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=c32615157847c6ad00128cef38e49033, regionState=CLOSING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:40:40,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, REOPEN/MOVE because future has completed 2024-11-13T10:40:40,598 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T10:40:40,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:40:40,751 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close c32615157847c6ad00128cef38e49033 2024-11-13T10:40:40,751 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T10:40:40,751 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing c32615157847c6ad00128cef38e49033, disabling compactions & flushes 2024-11-13T10:40:40,751 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:40,751 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:40,751 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. after waiting 0 ms 2024-11-13T10:40:40,751 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:40,756 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T10:40:40,757 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:40,757 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for c32615157847c6ad00128cef38e49033: Waiting for close lock at 1731494440751Running coprocessor pre-close hooks at 1731494440751Disabling compacts and flushes for region at 1731494440751Disabling writes for close at 1731494440751Writing region close event to WAL at 1731494440752 (+1 ms)Running coprocessor post-close hooks at 1731494440756 (+4 ms)Closed at 1731494440757 (+1 ms) 2024-11-13T10:40:40,757 WARN [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: c32615157847c6ad00128cef38e49033 to self. 2024-11-13T10:40:40,759 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed c32615157847c6ad00128cef38e49033 2024-11-13T10:40:40,759 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=c32615157847c6ad00128cef38e49033, regionState=CLOSED 2024-11-13T10:40:40,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:40:40,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-13T10:40:40,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 in 165 msec 2024-11-13T10:40:40,766 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, REOPEN/MOVE; state=CLOSED, location=770665a7984d,39101,1731494292956; forceNewPlan=false, retain=true 2024-11-13T10:40:40,918 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=c32615157847c6ad00128cef38e49033, regionState=OPENING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:40:40,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, REOPEN/MOVE because future has completed 2024-11-13T10:40:40,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:40:41,030 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T10:40:41,093 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,093 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} 2024-11-13T10:40:41,094 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,094 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T10:40:41,094 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,094 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,100 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,101 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32615157847c6ad00128cef38e49033 columnFamilyName A 2024-11-13T10:40:41,102 DEBUG [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:41,103 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(327): Store=c32615157847c6ad00128cef38e49033/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:40:41,103 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,106 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32615157847c6ad00128cef38e49033 columnFamilyName B 2024-11-13T10:40:41,106 DEBUG [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:41,106 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(327): Store=c32615157847c6ad00128cef38e49033/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:40:41,107 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,108 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c32615157847c6ad00128cef38e49033 columnFamilyName C 2024-11-13T10:40:41,108 DEBUG [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:41,109 INFO [StoreOpener-c32615157847c6ad00128cef38e49033-1 {}] regionserver.HStore(327): Store=c32615157847c6ad00128cef38e49033/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T10:40:41,109 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,110 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,111 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,112 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,112 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,112 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T10:40:41,115 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,117 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened c32615157847c6ad00128cef38e49033; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66422787, jitterRate=-0.010223343968391418}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T10:40:41,117 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,118 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for c32615157847c6ad00128cef38e49033: Running coprocessor pre-open hook at 1731494441094Writing region info on filesystem at 1731494441094Initializing all the Stores at 1731494441096 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494441097 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494441099 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731494441099Cleaning up temporary data from old regions at 1731494441112 (+13 ms)Running coprocessor post-open hooks at 1731494441117 (+5 ms)Region opened successfully at 1731494441118 (+1 ms) 2024-11-13T10:40:41,119 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., pid=153, masterSystemTime=1731494441090 2024-11-13T10:40:41,126 DEBUG [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,126 INFO [RS_OPEN_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,129 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=c32615157847c6ad00128cef38e49033, regionState=OPEN, openSeqNum=5, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:40:41,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-13T10:40:41,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 in 217 msec 2024-11-13T10:40:41,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-13T10:40:41,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, REOPEN/MOVE in 553 msec 2024-11-13T10:40:41,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-13T10:40:41,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 558 msec 2024-11-13T10:40:41,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 991 msec 2024-11-13T10:40:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-13T10:40:41,157 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:41,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:41,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:41,162 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36caa826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:41,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:41,163 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:41,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c5e6633, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:41,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:41,166 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a827c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:41,167 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54a80243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:41,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:41,169 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb93284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:41,172 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@133f8674, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:41,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:41,175 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349016b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:41,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:41,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:41,180 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0ebd5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:41,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:41,198 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:41,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@423f99d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:41,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:41,200 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c338e8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:41,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:41,204 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5042be83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:41,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:41,207 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:41,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8a13db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:41,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:41,216 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:41,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:41,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a487a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:41,223 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73bba0e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:41,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:41,240 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2fdc96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:41,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:41,243 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d06995, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:41,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:41,247 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54bbd5a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:41,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:41,249 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e65c827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 770665a7984d,44523,-1 for getting cluster id 2024-11-13T10:40:41,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T10:40:41,251 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f1b4553-470e-4e06-bc79-b9c06e16b8f8' 2024-11-13T10:40:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T10:40:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f1b4553-470e-4e06-bc79-b9c06e16b8f8" 2024-11-13T10:40:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5afc4bf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [770665a7984d,44523,-1] 2024-11-13T10:40:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T10:40:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:40:41,252 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T10:40:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7056c32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T10:40:41,253 DEBUG [Thread-2520 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:41,253 DEBUG [Thread-2535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:41,254 DEBUG [Thread-2520 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:41,255 DEBUG [Thread-2520 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:41,255 DEBUG [Thread-2522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:41,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:41,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:41,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:40:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:40:41,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:40:41,272 DEBUG [Thread-2537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:41,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:41,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:40:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,279 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:41,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:41,279 DEBUG [Thread-2528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:41,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:41,283 DEBUG [Thread-2531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:41,284 DEBUG [Thread-2531 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:41,285 DEBUG [Thread-2531 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:41,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:41,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:41,287 DEBUG [Thread-2524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:41,288 DEBUG [Thread-2524 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:40:41,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:40:41,289 DEBUG [Thread-2524 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:41,290 DEBUG [Thread-2526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:41,291 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:41,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:40:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:40:41,300 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-13T10:40:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T10:40:41,308 DEBUG [Thread-2533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:41,309 DEBUG [Thread-2533 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:41,309 DEBUG [Thread-2533 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:41,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:41,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:40:41,316 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:41,316 DEBUG [Thread-2539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T10:40:41,318 DEBUG [Thread-2539 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=770665a7984d,39101,1731494292956, seqNum=-1] 2024-11-13T10:40:41,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:41,318 DEBUG [Thread-2539 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T10:40:41,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:41,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T10:40:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5] 2024-11-13T10:40:41,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494501345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494501342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494501350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494501351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494501357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ed7a8637ce9e4028b9fa89e5815385ed_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494441278/Put/seqid=0 2024-11-13T10:40:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T10:40:41,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742394_1570 (size=12154) 2024-11-13T10:40:41,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494501466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494501467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494501467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494501467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494501467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,489 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T10:40:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T10:40:41,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494501677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494501677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494501678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494501680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494501681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:41,832 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ed7a8637ce9e4028b9fa89e5815385ed_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ed7a8637ce9e4028b9fa89e5815385ed_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:41,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/e008dc1d241441f18da4d425c1adc1ca, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:41,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/e008dc1d241441f18da4d425c1adc1ca is 175, key is test_row_0/A:col10/1731494441278/Put/seqid=0 2024-11-13T10:40:41,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742395_1571 (size=30955) 2024-11-13T10:40:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T10:40:41,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:41,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:41,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494501987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494501987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494501987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494501987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494501987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:42,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,277 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/e008dc1d241441f18da4d425c1adc1ca 2024-11-13T10:40:42,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/0afa181c2f6245529d2a06b05b2b2b67 is 50, key is test_row_0/B:col10/1731494441278/Put/seqid=0 2024-11-13T10:40:42,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742396_1572 (size=12001) 2024-11-13T10:40:42,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/0afa181c2f6245529d2a06b05b2b2b67 2024-11-13T10:40:42,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/101f86312add43e79d07b5fab3181724 is 50, key is test_row_0/C:col10/1731494441278/Put/seqid=0 2024-11-13T10:40:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T10:40:42,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742397_1573 (size=12001) 2024-11-13T10:40:42,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/101f86312add43e79d07b5fab3181724 2024-11-13T10:40:42,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/e008dc1d241441f18da4d425c1adc1ca as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/e008dc1d241441f18da4d425c1adc1ca 2024-11-13T10:40:42,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/e008dc1d241441f18da4d425c1adc1ca, entries=150, sequenceid=15, filesize=30.2 K 2024-11-13T10:40:42,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/0afa181c2f6245529d2a06b05b2b2b67 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0afa181c2f6245529d2a06b05b2b2b67 2024-11-13T10:40:42,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0afa181c2f6245529d2a06b05b2b2b67, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T10:40:42,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/101f86312add43e79d07b5fab3181724 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/101f86312add43e79d07b5fab3181724 2024-11-13T10:40:42,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494502497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:42,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:42,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494502497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494502497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:42,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:42,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:42,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494502497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:42,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/101f86312add43e79d07b5fab3181724, entries=150, sequenceid=15, filesize=11.7 K 2024-11-13T10:40:42,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c32615157847c6ad00128cef38e49033 in 1223ms, sequenceid=15, compaction requested=false 2024-11-13T10:40:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T10:40:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:42,512 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:42,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T10:40:42,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111337991e6bf5b149319336e0e99ed1ab15_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494441351/Put/seqid=0 2024-11-13T10:40:42,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494502554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:42,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742398_1574 (size=12154) 2024-11-13T10:40:42,576 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:42,581 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111337991e6bf5b149319336e0e99ed1ab15_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111337991e6bf5b149319336e0e99ed1ab15_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:42,584 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/b66af2fe76624f7f83449f592ead6bf8, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:42,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/b66af2fe76624f7f83449f592ead6bf8 is 175, key is test_row_0/A:col10/1731494441351/Put/seqid=0 2024-11-13T10:40:42,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742399_1575 (size=30955) 2024-11-13T10:40:42,609 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/b66af2fe76624f7f83449f592ead6bf8 2024-11-13T10:40:42,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/373164ad097042ac9a5c5760fa8526bd is 50, key is test_row_0/B:col10/1731494441351/Put/seqid=0 2024-11-13T10:40:42,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494502667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:42,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742400_1576 (size=12001) 2024-11-13T10:40:42,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:42,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:42,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:42,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:42,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494502877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:43,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:43,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:43,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:43,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:43,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:43,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:43,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/373164ad097042ac9a5c5760fa8526bd 2024-11-13T10:40:43,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/d6d45f7d799e4bccaf812f24c522be58 is 50, key is test_row_0/C:col10/1731494441351/Put/seqid=0 2024-11-13T10:40:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742401_1577 (size=12001) 2024-11-13T10:40:43,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/d6d45f7d799e4bccaf812f24c522be58 2024-11-13T10:40:43,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/b66af2fe76624f7f83449f592ead6bf8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/b66af2fe76624f7f83449f592ead6bf8 2024-11-13T10:40:43,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/b66af2fe76624f7f83449f592ead6bf8, entries=150, sequenceid=41, filesize=30.2 K 2024-11-13T10:40:43,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/373164ad097042ac9a5c5760fa8526bd as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/373164ad097042ac9a5c5760fa8526bd 2024-11-13T10:40:43,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/373164ad097042ac9a5c5760fa8526bd, entries=150, sequenceid=41, filesize=11.7 K 2024-11-13T10:40:43,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/d6d45f7d799e4bccaf812f24c522be58 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/d6d45f7d799e4bccaf812f24c522be58 2024-11-13T10:40:43,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/d6d45f7d799e4bccaf812f24c522be58, entries=150, sequenceid=41, filesize=11.7 K 2024-11-13T10:40:43,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c32615157847c6ad00128cef38e49033 in 677ms, sequenceid=41, compaction requested=false 2024-11-13T10:40:43,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:43,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T10:40:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:43,190 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T10:40:43,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:43,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111338497f674c28401d99afee813b6ac26e_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494442515/Put/seqid=0 2024-11-13T10:40:43,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742402_1578 (size=12154) 2024-11-13T10:40:43,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494503430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:43,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T10:40:43,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494503508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:43,514 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in c32615157847c6ad00128cef38e49033 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:43,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:43,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494503517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:43,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494503517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:43,525 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_2' in c32615157847c6ad00128cef38e49033 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494503517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:43,526 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in c32615157847c6ad00128cef38e49033 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:43,526 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_1' in c32615157847c6ad00128cef38e49033 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:43,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494503546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:43,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:43,678 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111338497f674c28401d99afee813b6ac26e_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111338497f674c28401d99afee813b6ac26e_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:43,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/077afe01918142deadfa5f10efc7c7a0, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:43,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/077afe01918142deadfa5f10efc7c7a0 is 175, key is test_row_0/A:col10/1731494442515/Put/seqid=0 2024-11-13T10:40:43,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742403_1579 (size=30955) 2024-11-13T10:40:43,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494503761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494504077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:44,121 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/077afe01918142deadfa5f10efc7c7a0 2024-11-13T10:40:44,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/120a95935e704a3a92932504803984a4 is 50, key is test_row_0/B:col10/1731494442515/Put/seqid=0 2024-11-13T10:40:44,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742404_1580 (size=12001) 2024-11-13T10:40:44,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494504600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:44,609 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/120a95935e704a3a92932504803984a4 2024-11-13T10:40:44,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/3ecc0874c1284833b53fa9378b9f476a is 50, key is test_row_0/C:col10/1731494442515/Put/seqid=0 2024-11-13T10:40:44,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742405_1581 (size=12001) 2024-11-13T10:40:45,087 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/3ecc0874c1284833b53fa9378b9f476a 2024-11-13T10:40:45,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/077afe01918142deadfa5f10efc7c7a0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/077afe01918142deadfa5f10efc7c7a0 2024-11-13T10:40:45,097 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/077afe01918142deadfa5f10efc7c7a0, entries=150, sequenceid=51, filesize=30.2 K 2024-11-13T10:40:45,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/120a95935e704a3a92932504803984a4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/120a95935e704a3a92932504803984a4 2024-11-13T10:40:45,101 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/120a95935e704a3a92932504803984a4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T10:40:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/3ecc0874c1284833b53fa9378b9f476a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/3ecc0874c1284833b53fa9378b9f476a 2024-11-13T10:40:45,105 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/3ecc0874c1284833b53fa9378b9f476a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T10:40:45,106 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for c32615157847c6ad00128cef38e49033 in 1916ms, sequenceid=51, compaction requested=true 2024-11-13T10:40:45,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:45,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:45,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-13T10:40:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-13T10:40:45,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-13T10:40:45,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7890 sec 2024-11-13T10:40:45,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.8100 sec 2024-11-13T10:40:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T10:40:45,467 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:45,468 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-13T10:40:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T10:40:45,474 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:45,475 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:45,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:45,533 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:45,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T10:40:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:45,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111352b7ac2f1c8e49b6bc378a36e11c114c_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494443386/Put/seqid=0 2024-11-13T10:40:45,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494505556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742406_1582 (size=14594) 2024-11-13T10:40:45,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494505564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494505564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,573 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:45,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494505565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T10:40:45,587 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111352b7ac2f1c8e49b6bc378a36e11c114c_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111352b7ac2f1c8e49b6bc378a36e11c114c_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:45,602 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/105ecc8c40484fcebe0c0395d9415774, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:45,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/105ecc8c40484fcebe0c0395d9415774 is 175, key is test_row_0/A:col10/1731494443386/Put/seqid=0 2024-11-13T10:40:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494505620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,624 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in c32615157847c6ad00128cef38e49033 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:45,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:45,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:45,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:45,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:45,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:45,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742407_1583 (size=39549) 2024-11-13T10:40:45,636 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/105ecc8c40484fcebe0c0395d9415774 2024-11-13T10:40:45,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/9b6632afcf41449582b3412e66b97c9d is 50, key is test_row_0/B:col10/1731494443386/Put/seqid=0 2024-11-13T10:40:45,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742408_1584 (size=12001) 2024-11-13T10:40:45,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/9b6632afcf41449582b3412e66b97c9d 2024-11-13T10:40:45,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/97ace567baf644cabb91e5fc52df2345 is 50, key is test_row_0/C:col10/1731494443386/Put/seqid=0 2024-11-13T10:40:45,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742409_1585 (size=12001) 2024-11-13T10:40:45,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494505677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494505677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494505677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494505677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:45,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:45,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:45,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:45,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:45,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T10:40:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494505887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494505887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494505889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494505892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:45,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:45,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:45,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:45,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:45,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:45,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/97ace567baf644cabb91e5fc52df2345 2024-11-13T10:40:46,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/105ecc8c40484fcebe0c0395d9415774 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/105ecc8c40484fcebe0c0395d9415774 2024-11-13T10:40:46,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/105ecc8c40484fcebe0c0395d9415774, entries=200, sequenceid=78, filesize=38.6 K 2024-11-13T10:40:46,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:46,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/9b6632afcf41449582b3412e66b97c9d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/9b6632afcf41449582b3412e66b97c9d 2024-11-13T10:40:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T10:40:46,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/9b6632afcf41449582b3412e66b97c9d, entries=150, sequenceid=78, filesize=11.7 K 2024-11-13T10:40:46,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/97ace567baf644cabb91e5fc52df2345 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/97ace567baf644cabb91e5fc52df2345 2024-11-13T10:40:46,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/97ace567baf644cabb91e5fc52df2345, entries=150, sequenceid=78, filesize=11.7 K 2024-11-13T10:40:46,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for c32615157847c6ad00128cef38e49033 in 591ms, sequenceid=78, compaction requested=true 2024-11-13T10:40:46,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:46,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:46,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:46,125 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:46,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:46,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:46,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:46,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:46,125 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:46,126 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:46,126 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/A is initiating minor compaction (all files) 2024-11-13T10:40:46,126 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/A in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,126 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:46,126 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/B is initiating minor compaction (all files) 2024-11-13T10:40:46,126 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/e008dc1d241441f18da4d425c1adc1ca, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/b66af2fe76624f7f83449f592ead6bf8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/077afe01918142deadfa5f10efc7c7a0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/105ecc8c40484fcebe0c0395d9415774] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=129.3 K 2024-11-13T10:40:46,126 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/B in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,126 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,126 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0afa181c2f6245529d2a06b05b2b2b67, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/373164ad097042ac9a5c5760fa8526bd, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/120a95935e704a3a92932504803984a4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/9b6632afcf41449582b3412e66b97c9d] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=46.9 K 2024-11-13T10:40:46,126 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/e008dc1d241441f18da4d425c1adc1ca, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/b66af2fe76624f7f83449f592ead6bf8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/077afe01918142deadfa5f10efc7c7a0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/105ecc8c40484fcebe0c0395d9415774] 2024-11-13T10:40:46,127 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0afa181c2f6245529d2a06b05b2b2b67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731494441271 2024-11-13T10:40:46,127 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting e008dc1d241441f18da4d425c1adc1ca, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731494441271 2024-11-13T10:40:46,127 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 373164ad097042ac9a5c5760fa8526bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731494441344 2024-11-13T10:40:46,128 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b66af2fe76624f7f83449f592ead6bf8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731494441344 2024-11-13T10:40:46,137 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 120a95935e704a3a92932504803984a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731494442515 2024-11-13T10:40:46,137 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 077afe01918142deadfa5f10efc7c7a0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731494442515 2024-11-13T10:40:46,140 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b6632afcf41449582b3412e66b97c9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731494443386 2024-11-13T10:40:46,141 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 105ecc8c40484fcebe0c0395d9415774, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731494443359 2024-11-13T10:40:46,189 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:46,190 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#B#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:46,191 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/732bd74439cd4afeb42a1c7e5de5d763 is 50, key is test_row_0/B:col10/1731494443386/Put/seqid=0 2024-11-13T10:40:46,198 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113cae4ddbd83884641a245591787896cbe_c32615157847c6ad00128cef38e49033 store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:46,201 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:46,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T10:40:46,201 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113cae4ddbd83884641a245591787896cbe_c32615157847c6ad00128cef38e49033, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:46,201 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cae4ddbd83884641a245591787896cbe_c32615157847c6ad00128cef38e49033 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:46,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742410_1586 (size=12139) 2024-11-13T10:40:46,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742411_1587 (size=4469) 2024-11-13T10:40:46,206 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#A#compaction#499 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:46,207 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/44dbd5a9d945467385465f5a21b04f67 is 175, key is test_row_0/A:col10/1731494443386/Put/seqid=0 2024-11-13T10:40:46,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742412_1588 (size=31093) 2024-11-13T10:40:46,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111384c80b3e69da4d4698250c145b2f72b2_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494446200/Put/seqid=0 2024-11-13T10:40:46,218 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/44dbd5a9d945467385465f5a21b04f67 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/44dbd5a9d945467385465f5a21b04f67 2024-11-13T10:40:46,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742413_1589 (size=14594) 2024-11-13T10:40:46,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:46,223 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c32615157847c6ad00128cef38e49033/A of c32615157847c6ad00128cef38e49033 into 44dbd5a9d945467385465f5a21b04f67(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:46,223 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:46,223 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/A, priority=12, startTime=1731494446125; duration=0sec 2024-11-13T10:40:46,223 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:46,223 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:A 2024-11-13T10:40:46,223 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:40:46,224 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111384c80b3e69da4d4698250c145b2f72b2_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111384c80b3e69da4d4698250c145b2f72b2_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:46,225 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:40:46,225 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/C is initiating minor compaction (all files) 2024-11-13T10:40:46,225 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/C in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,225 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/101f86312add43e79d07b5fab3181724, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/d6d45f7d799e4bccaf812f24c522be58, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/3ecc0874c1284833b53fa9378b9f476a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/97ace567baf644cabb91e5fc52df2345] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=46.9 K 2024-11-13T10:40:46,225 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/946db4be148a4c0da65a6c2b448ea3d3, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:46,225 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 101f86312add43e79d07b5fab3181724, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731494441271 2024-11-13T10:40:46,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/946db4be148a4c0da65a6c2b448ea3d3 is 175, key is test_row_0/A:col10/1731494446200/Put/seqid=0 2024-11-13T10:40:46,226 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6d45f7d799e4bccaf812f24c522be58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731494441344 2024-11-13T10:40:46,228 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ecc0874c1284833b53fa9378b9f476a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731494442515 2024-11-13T10:40:46,229 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97ace567baf644cabb91e5fc52df2345, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731494443386 2024-11-13T10:40:46,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742414_1590 (size=39549) 2024-11-13T10:40:46,233 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/946db4be148a4c0da65a6c2b448ea3d3 2024-11-13T10:40:46,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/b372e2c3f80e41c5b7ff9c3439e2268c is 50, key is test_row_0/B:col10/1731494446200/Put/seqid=0 2024-11-13T10:40:46,241 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#C#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:46,241 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/e83b736795b5407689af646a48fd39be is 50, key is test_row_0/C:col10/1731494443386/Put/seqid=0 2024-11-13T10:40:46,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494506255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494506257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494506266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742415_1591 (size=12001) 2024-11-13T10:40:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494506268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/b372e2c3f80e41c5b7ff9c3439e2268c 2024-11-13T10:40:46,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742416_1592 (size=12139) 2024-11-13T10:40:46,305 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/e83b736795b5407689af646a48fd39be as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/e83b736795b5407689af646a48fd39be 2024-11-13T10:40:46,310 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c32615157847c6ad00128cef38e49033/C of c32615157847c6ad00128cef38e49033 into e83b736795b5407689af646a48fd39be(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:46,310 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:46,310 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/C, priority=12, startTime=1731494446125; duration=0sec 2024-11-13T10:40:46,310 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:46,310 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:C 2024-11-13T10:40:46,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/ff1a537170224fdb853ac49e4e6a5b16 is 50, key is test_row_0/C:col10/1731494446200/Put/seqid=0 2024-11-13T10:40:46,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742417_1593 (size=12001) 2024-11-13T10:40:46,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/ff1a537170224fdb853ac49e4e6a5b16 2024-11-13T10:40:46,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/946db4be148a4c0da65a6c2b448ea3d3 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/946db4be148a4c0da65a6c2b448ea3d3 2024-11-13T10:40:46,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/946db4be148a4c0da65a6c2b448ea3d3, entries=200, sequenceid=91, filesize=38.6 K 2024-11-13T10:40:46,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/b372e2c3f80e41c5b7ff9c3439e2268c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b372e2c3f80e41c5b7ff9c3439e2268c 2024-11-13T10:40:46,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b372e2c3f80e41c5b7ff9c3439e2268c, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T10:40:46,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/ff1a537170224fdb853ac49e4e6a5b16 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/ff1a537170224fdb853ac49e4e6a5b16 2024-11-13T10:40:46,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/ff1a537170224fdb853ac49e4e6a5b16, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T10:40:46,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for c32615157847c6ad00128cef38e49033 in 160ms, sequenceid=91, compaction requested=false 2024-11-13T10:40:46,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:46,385 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:46,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T10:40:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:46,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494506408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494506410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494506416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494506414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111314714183bc9143198c4fa3a0ea408c8d_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494446252/Put/seqid=0 2024-11-13T10:40:46,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742418_1594 (size=12154) 2024-11-13T10:40:46,473 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:46,484 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111314714183bc9143198c4fa3a0ea408c8d_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111314714183bc9143198c4fa3a0ea408c8d_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:46,492 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/1e8a3167a69a4132a965f797151a6daf, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:46,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/1e8a3167a69a4132a965f797151a6daf is 175, key is test_row_0/A:col10/1731494446252/Put/seqid=0 2024-11-13T10:40:46,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742419_1595 (size=30955) 2024-11-13T10:40:46,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494506529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494506529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494506530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494506540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:46,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:46,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,601 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T10:40:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T10:40:46,631 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/732bd74439cd4afeb42a1c7e5de5d763 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/732bd74439cd4afeb42a1c7e5de5d763 2024-11-13T10:40:46,661 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c32615157847c6ad00128cef38e49033/B of c32615157847c6ad00128cef38e49033 into 732bd74439cd4afeb42a1c7e5de5d763(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:46,661 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:46,661 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/B, priority=12, startTime=1731494446125; duration=0sec 2024-11-13T10:40:46,661 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:46,661 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:B 2024-11-13T10:40:46,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:46,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:46,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494506737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494506737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494506747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494506756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:46,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:46,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:46,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:46,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:46,931 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/1e8a3167a69a4132a965f797151a6daf 2024-11-13T10:40:46,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/a109857961864cdc9d9d66756298b92c is 50, key is test_row_0/B:col10/1731494446252/Put/seqid=0 2024-11-13T10:40:46,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742420_1596 (size=12001) 2024-11-13T10:40:46,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/a109857961864cdc9d9d66756298b92c 2024-11-13T10:40:46,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/6981362edf644397aa193203b6d92a0c is 50, key is test_row_0/C:col10/1731494446252/Put/seqid=0 2024-11-13T10:40:46,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742421_1597 (size=12001) 2024-11-13T10:40:47,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:47,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:47,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:47,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:47,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494507047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494507047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494507057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494507066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:47,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:47,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:47,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:47,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:47,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:47,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:47,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:47,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/6981362edf644397aa193203b6d92a0c 2024-11-13T10:40:47,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/1e8a3167a69a4132a965f797151a6daf as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1e8a3167a69a4132a965f797151a6daf 2024-11-13T10:40:47,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1e8a3167a69a4132a965f797151a6daf, entries=150, sequenceid=119, filesize=30.2 K 2024-11-13T10:40:47,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/a109857961864cdc9d9d66756298b92c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a109857961864cdc9d9d66756298b92c 2024-11-13T10:40:47,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a109857961864cdc9d9d66756298b92c, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T10:40:47,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/6981362edf644397aa193203b6d92a0c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6981362edf644397aa193203b6d92a0c 2024-11-13T10:40:47,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6981362edf644397aa193203b6d92a0c, entries=150, sequenceid=119, filesize=11.7 K 2024-11-13T10:40:47,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c32615157847c6ad00128cef38e49033 in 1004ms, sequenceid=119, compaction requested=true 2024-11-13T10:40:47,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:47,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:47,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:47,389 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:47,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:47,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:47,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:47,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:47,389 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:47,393 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:47,393 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/B is initiating minor compaction (all files) 2024-11-13T10:40:47,393 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/B in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,393 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/732bd74439cd4afeb42a1c7e5de5d763, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b372e2c3f80e41c5b7ff9c3439e2268c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a109857961864cdc9d9d66756298b92c] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=35.3 K 2024-11-13T10:40:47,394 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 732bd74439cd4afeb42a1c7e5de5d763, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731494443386 2024-11-13T10:40:47,394 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting b372e2c3f80e41c5b7ff9c3439e2268c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731494445562 2024-11-13T10:40:47,394 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting a109857961864cdc9d9d66756298b92c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494446252 2024-11-13T10:40:47,399 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:47,399 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/A is initiating minor compaction (all files) 2024-11-13T10:40:47,399 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/A in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,399 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/44dbd5a9d945467385465f5a21b04f67, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/946db4be148a4c0da65a6c2b448ea3d3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1e8a3167a69a4132a965f797151a6daf] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=99.2 K 2024-11-13T10:40:47,399 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,399 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/44dbd5a9d945467385465f5a21b04f67, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/946db4be148a4c0da65a6c2b448ea3d3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1e8a3167a69a4132a965f797151a6daf] 2024-11-13T10:40:47,400 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44dbd5a9d945467385465f5a21b04f67, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731494443386 2024-11-13T10:40:47,404 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 946db4be148a4c0da65a6c2b448ea3d3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731494445562 2024-11-13T10:40:47,405 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e8a3167a69a4132a965f797151a6daf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494446252 2024-11-13T10:40:47,420 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#B#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:47,421 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/2e32706acb0c4a19b007d51afba5a461 is 50, key is test_row_0/B:col10/1731494446252/Put/seqid=0 2024-11-13T10:40:47,422 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:47,424 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111365a06460b3614cf5b298fb14b69f43c7_c32615157847c6ad00128cef38e49033 store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:47,426 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111365a06460b3614cf5b298fb14b69f43c7_c32615157847c6ad00128cef38e49033, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:47,427 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111365a06460b3614cf5b298fb14b69f43c7_c32615157847c6ad00128cef38e49033 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:47,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742423_1599 (size=4469) 2024-11-13T10:40:47,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742422_1598 (size=12241) 2024-11-13T10:40:47,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T10:40:47,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,495 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T10:40:47,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138907141e621149b69af33e69f8101f50_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494446409/Put/seqid=0 2024-11-13T10:40:47,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742424_1600 (size=12154) 2024-11-13T10:40:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:47,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T10:40:47,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494507628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494507627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494507629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494507635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494507646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,649 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in c32615157847c6ad00128cef38e49033 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494507737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494507737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494507737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494507748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,866 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#A#compaction#508 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:47,867 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/dcc169d9e192417fbf758825d8879e8f is 175, key is test_row_0/A:col10/1731494446252/Put/seqid=0 2024-11-13T10:40:47,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742425_1601 (size=31195) 2024-11-13T10:40:47,899 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/2e32706acb0c4a19b007d51afba5a461 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e32706acb0c4a19b007d51afba5a461 2024-11-13T10:40:47,903 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/B of c32615157847c6ad00128cef38e49033 into 2e32706acb0c4a19b007d51afba5a461(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:47,903 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:47,903 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/B, priority=13, startTime=1731494447389; duration=0sec 2024-11-13T10:40:47,903 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:47,903 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:B 2024-11-13T10:40:47,903 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:47,904 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:47,904 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/C is initiating minor compaction (all files) 2024-11-13T10:40:47,904 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/C in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:47,904 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/e83b736795b5407689af646a48fd39be, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/ff1a537170224fdb853ac49e4e6a5b16, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6981362edf644397aa193203b6d92a0c] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=35.3 K 2024-11-13T10:40:47,905 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting e83b736795b5407689af646a48fd39be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731494443386 2024-11-13T10:40:47,905 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1a537170224fdb853ac49e4e6a5b16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731494445562 2024-11-13T10:40:47,905 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6981362edf644397aa193203b6d92a0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494446252 2024-11-13T10:40:47,911 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#C#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:47,912 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/c443665f26f24d028d41a4e068b9a457 is 50, key is test_row_0/C:col10/1731494446252/Put/seqid=0 2024-11-13T10:40:47,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742426_1602 (size=12241) 2024-11-13T10:40:47,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:47,924 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/c443665f26f24d028d41a4e068b9a457 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c443665f26f24d028d41a4e068b9a457 2024-11-13T10:40:47,931 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138907141e621149b69af33e69f8101f50_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138907141e621149b69af33e69f8101f50_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:47,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/5ba20a20a88d4cd2a449da52c50e2100, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:47,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/5ba20a20a88d4cd2a449da52c50e2100 is 175, key is test_row_0/A:col10/1731494446409/Put/seqid=0 2024-11-13T10:40:47,934 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/C of c32615157847c6ad00128cef38e49033 into c443665f26f24d028d41a4e068b9a457(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:47,934 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:47,934 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/C, priority=13, startTime=1731494447389; duration=0sec 2024-11-13T10:40:47,934 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:47,934 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:C 2024-11-13T10:40:47,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742427_1603 (size=30955) 2024-11-13T10:40:47,939 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/5ba20a20a88d4cd2a449da52c50e2100 2024-11-13T10:40:47,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494507947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494507947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494507947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/0f75245051b64347aef1f3a375c44162 is 50, key is test_row_0/B:col10/1731494446409/Put/seqid=0 2024-11-13T10:40:47,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494507956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:47,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742428_1604 (size=12001) 2024-11-13T10:40:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494508257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:48,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494508257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:48,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494508257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:48,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494508268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:48,300 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/dcc169d9e192417fbf758825d8879e8f as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/dcc169d9e192417fbf758825d8879e8f 2024-11-13T10:40:48,316 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/A of c32615157847c6ad00128cef38e49033 into dcc169d9e192417fbf758825d8879e8f(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:48,316 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:48,316 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/A, priority=13, startTime=1731494447389; duration=0sec 2024-11-13T10:40:48,316 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:48,317 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:A 2024-11-13T10:40:48,390 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/0f75245051b64347aef1f3a375c44162 2024-11-13T10:40:48,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/b25a99571620456e8c22cf5cf27d8cc4 is 50, key is test_row_0/C:col10/1731494446409/Put/seqid=0 2024-11-13T10:40:48,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742429_1605 (size=12001) 2024-11-13T10:40:48,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494508767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494508767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:48,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494508767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494508777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:48,848 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/b25a99571620456e8c22cf5cf27d8cc4 2024-11-13T10:40:48,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/5ba20a20a88d4cd2a449da52c50e2100 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/5ba20a20a88d4cd2a449da52c50e2100 2024-11-13T10:40:48,864 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/5ba20a20a88d4cd2a449da52c50e2100, entries=150, sequenceid=130, filesize=30.2 K 2024-11-13T10:40:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/0f75245051b64347aef1f3a375c44162 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0f75245051b64347aef1f3a375c44162 2024-11-13T10:40:48,871 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0f75245051b64347aef1f3a375c44162, entries=150, sequenceid=130, filesize=11.7 K 2024-11-13T10:40:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/b25a99571620456e8c22cf5cf27d8cc4 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b25a99571620456e8c22cf5cf27d8cc4 2024-11-13T10:40:48,876 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b25a99571620456e8c22cf5cf27d8cc4, entries=150, sequenceid=130, filesize=11.7 K 2024-11-13T10:40:48,877 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for c32615157847c6ad00128cef38e49033 in 1383ms, sequenceid=130, compaction requested=false 2024-11-13T10:40:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-13T10:40:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-13T10:40:48,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-13T10:40:48,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4100 sec 2024-11-13T10:40:48,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.4200 sec 2024-11-13T10:40:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T10:40:49,637 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:49,637 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-13T10:40:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T10:40:49,642 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:49,642 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:49,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T10:40:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:49,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:49,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T10:40:49,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494509789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:49,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c59db86178514109be6865adc3f82a7b_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494447627/Put/seqid=0 2024-11-13T10:40:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:49,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T10:40:49,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:49,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:49,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:49,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:49,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:49,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494509792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:49,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:49,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494509794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:49,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494509796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:49,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:49,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742430_1606 (size=17284) 2024-11-13T10:40:49,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494509896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:49,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494509907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:49,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494509908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:49,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494509908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:49,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:49,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T10:40:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:49,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T10:40:50,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T10:40:50,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:50,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494510108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:50,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494510117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494510117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:50,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494510117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:50,202 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,205 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c59db86178514109be6865adc3f82a7b_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c59db86178514109be6865adc3f82a7b_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:50,206 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/0f02b344ad6349d9891c9e5264296bc6, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:50,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/0f02b344ad6349d9891c9e5264296bc6 is 175, key is test_row_0/A:col10/1731494447627/Put/seqid=0 2024-11-13T10:40:50,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742431_1607 (size=48389) 2024-11-13T10:40:50,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/0f02b344ad6349d9891c9e5264296bc6 2024-11-13T10:40:50,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/5e754925332a4caab78500fcb5751d73 is 50, key is test_row_0/B:col10/1731494447627/Put/seqid=0 2024-11-13T10:40:50,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742432_1608 (size=12151) 2024-11-13T10:40:50,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T10:40:50,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:50,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T10:40:50,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T10:40:50,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:50,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:50,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494510416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:50,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:50,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494510427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:50,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494510427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:50,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494510427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:50,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:50,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T10:40:50,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:50,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:50,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/5e754925332a4caab78500fcb5751d73 2024-11-13T10:40:50,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/5f68a5efc0024beba33e8b48abde1fe8 is 50, key is test_row_0/C:col10/1731494447627/Put/seqid=0 2024-11-13T10:40:50,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742433_1609 (size=12151) 2024-11-13T10:40:50,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/5f68a5efc0024beba33e8b48abde1fe8 2024-11-13T10:40:50,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/0f02b344ad6349d9891c9e5264296bc6 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/0f02b344ad6349d9891c9e5264296bc6 2024-11-13T10:40:50,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/0f02b344ad6349d9891c9e5264296bc6, entries=250, sequenceid=161, filesize=47.3 K 2024-11-13T10:40:50,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/5e754925332a4caab78500fcb5751d73 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e754925332a4caab78500fcb5751d73 2024-11-13T10:40:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e754925332a4caab78500fcb5751d73, entries=150, sequenceid=161, filesize=11.9 K 2024-11-13T10:40:50,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/5f68a5efc0024beba33e8b48abde1fe8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/5f68a5efc0024beba33e8b48abde1fe8 2024-11-13T10:40:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/5f68a5efc0024beba33e8b48abde1fe8, entries=150, sequenceid=161, filesize=11.9 K 2024-11-13T10:40:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for c32615157847c6ad00128cef38e49033 in 913ms, sequenceid=161, compaction requested=true 2024-11-13T10:40:50,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,695 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:50,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:50,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,695 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:50,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:50,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:50,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:50,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,697 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110539 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:50,697 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/A is initiating minor compaction (all files) 2024-11-13T10:40:50,697 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/A in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,697 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/dcc169d9e192417fbf758825d8879e8f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/5ba20a20a88d4cd2a449da52c50e2100, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/0f02b344ad6349d9891c9e5264296bc6] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=107.9 K 2024-11-13T10:40:50,697 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,697 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/dcc169d9e192417fbf758825d8879e8f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/5ba20a20a88d4cd2a449da52c50e2100, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/0f02b344ad6349d9891c9e5264296bc6] 2024-11-13T10:40:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,698 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:50,698 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/B is initiating minor compaction (all files) 2024-11-13T10:40:50,698 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/B in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,698 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e32706acb0c4a19b007d51afba5a461, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0f75245051b64347aef1f3a375c44162, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e754925332a4caab78500fcb5751d73] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=35.5 K 2024-11-13T10:40:50,699 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting dcc169d9e192417fbf758825d8879e8f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494446252 2024-11-13T10:40:50,699 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e32706acb0c4a19b007d51afba5a461, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494446252 2024-11-13T10:40:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,699 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f75245051b64347aef1f3a375c44162, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494446408 2024-11-13T10:40:50,699 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba20a20a88d4cd2a449da52c50e2100, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494446408 2024-11-13T10:40:50,700 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e754925332a4caab78500fcb5751d73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494447627 2024-11-13T10:40:50,700 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f02b344ad6349d9891c9e5264296bc6, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494447623 2024-11-13T10:40:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,707 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:50,709 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#B#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:50,710 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/99a3a58b9ee14df2bd0cd1904457018e is 50, key is test_row_0/B:col10/1731494447627/Put/seqid=0 2024-11-13T10:40:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,712 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ebca00f6643b40838ba4102d70c0d5e9_c32615157847c6ad00128cef38e49033 store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,715 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ebca00f6643b40838ba4102d70c0d5e9_c32615157847c6ad00128cef38e49033, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:50,715 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ebca00f6643b40838ba4102d70c0d5e9_c32615157847c6ad00128cef38e49033 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T10:40:50,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,721 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-13T10:40:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742434_1610 (size=12493) 2024-11-13T10:40:50,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136c9298ee457a4147baf71a0c1a9ba6da_c32615157847c6ad00128cef38e49033 is 50, key is test_row_1/A:col10/1731494449791/Put/seqid=0 2024-11-13T10:40:50,744 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/99a3a58b9ee14df2bd0cd1904457018e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/99a3a58b9ee14df2bd0cd1904457018e 2024-11-13T10:40:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,749 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/B of c32615157847c6ad00128cef38e49033 into 99a3a58b9ee14df2bd0cd1904457018e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:50,749 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:50,749 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/B, priority=13, startTime=1731494450695; duration=0sec 2024-11-13T10:40:50,749 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:50,749 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:B 2024-11-13T10:40:50,749 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,750 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:50,750 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/C is initiating minor compaction (all files) 2024-11-13T10:40:50,750 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/C in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:50,750 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c443665f26f24d028d41a4e068b9a457, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b25a99571620456e8c22cf5cf27d8cc4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/5f68a5efc0024beba33e8b48abde1fe8] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=35.5 K 2024-11-13T10:40:50,751 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting c443665f26f24d028d41a4e068b9a457, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731494446252 2024-11-13T10:40:50,751 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting b25a99571620456e8c22cf5cf27d8cc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731494446408 2024-11-13T10:40:50,752 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f68a5efc0024beba33e8b48abde1fe8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494447627 2024-11-13T10:40:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742435_1611 (size=4469) 2024-11-13T10:40:50,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,754 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#A#compaction#517 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,755 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/a839e5f08bf34be58a6d7c156f23ef7a is 175, key is test_row_0/A:col10/1731494447627/Put/seqid=0 2024-11-13T10:40:50,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T10:40:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,779 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#C#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,779 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/c0672ad649c44a76a30875c538154c09 is 50, key is test_row_0/C:col10/1731494447627/Put/seqid=0 2024-11-13T10:40:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742436_1612 (size=9814) 2024-11-13T10:40:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742437_1613 (size=31447) 2024-11-13T10:40:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742438_1614 (size=12493) 2024-11-13T10:40:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:50,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:51,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494511066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494511066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494511068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494511069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494511177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494511177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494511177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494511177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:51,198 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136c9298ee457a4147baf71a0c1a9ba6da_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136c9298ee457a4147baf71a0c1a9ba6da_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/daefd0d0cb7f4f11b774da94383f41cc, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/daefd0d0cb7f4f11b774da94383f41cc is 175, key is test_row_1/A:col10/1731494449791/Put/seqid=0 2024-11-13T10:40:51,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742439_1615 (size=22461) 2024-11-13T10:40:51,236 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/a839e5f08bf34be58a6d7c156f23ef7a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/a839e5f08bf34be58a6d7c156f23ef7a 2024-11-13T10:40:51,242 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/A of c32615157847c6ad00128cef38e49033 into a839e5f08bf34be58a6d7c156f23ef7a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:51,242 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:51,242 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/A, priority=13, startTime=1731494450694; duration=0sec 2024-11-13T10:40:51,242 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:51,242 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:A 2024-11-13T10:40:51,259 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/c0672ad649c44a76a30875c538154c09 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c0672ad649c44a76a30875c538154c09 2024-11-13T10:40:51,268 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/C of c32615157847c6ad00128cef38e49033 into c0672ad649c44a76a30875c538154c09(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:51,268 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:51,268 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/C, priority=13, startTime=1731494450695; duration=0sec 2024-11-13T10:40:51,268 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:51,268 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:C 2024-11-13T10:40:51,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494511386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494511387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494511387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494511387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,608 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/daefd0d0cb7f4f11b774da94383f41cc 2024-11-13T10:40:51,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/5e12594e21f9469a97827791cbed6378 is 50, key is test_row_1/B:col10/1731494449791/Put/seqid=0 2024-11-13T10:40:51,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742440_1616 (size=9757) 2024-11-13T10:40:51,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45572 deadline: 1731494511667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,669 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in c32615157847c6ad00128cef38e49033 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494511697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494511697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494511700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494511701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T10:40:52,043 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/5e12594e21f9469a97827791cbed6378 2024-11-13T10:40:52,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/bd3328d2e57f43a88b06cef0d649a14c is 50, key is test_row_1/C:col10/1731494449791/Put/seqid=0 2024-11-13T10:40:52,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742441_1617 (size=9757) 2024-11-13T10:40:52,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494512207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:52,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494512207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:52,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494512207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:52,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:52,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:52,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:52,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494512216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:52,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:52,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:52,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:52,462 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/bd3328d2e57f43a88b06cef0d649a14c 2024-11-13T10:40:52,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/daefd0d0cb7f4f11b774da94383f41cc as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/daefd0d0cb7f4f11b774da94383f41cc 2024-11-13T10:40:52,470 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/daefd0d0cb7f4f11b774da94383f41cc, entries=100, sequenceid=170, filesize=21.9 K 2024-11-13T10:40:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/5e12594e21f9469a97827791cbed6378 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e12594e21f9469a97827791cbed6378 2024-11-13T10:40:52,479 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e12594e21f9469a97827791cbed6378, entries=100, sequenceid=170, filesize=9.5 K 2024-11-13T10:40:52,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/bd3328d2e57f43a88b06cef0d649a14c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/bd3328d2e57f43a88b06cef0d649a14c 2024-11-13T10:40:52,483 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/bd3328d2e57f43a88b06cef0d649a14c, entries=100, sequenceid=170, filesize=9.5 K 2024-11-13T10:40:52,484 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for c32615157847c6ad00128cef38e49033 in 1763ms, sequenceid=170, compaction requested=false 2024-11-13T10:40:52,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:52,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:52,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-13T10:40:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-13T10:40:52,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-13T10:40:52,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8440 sec 2024-11-13T10:40:52,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.8510 sec 2024-11-13T10:40:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:53,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:53,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-13T10:40:53,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130ad1e3abc7824ea4bd0fcab5f6a7c39e_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494451066/Put/seqid=0 2024-11-13T10:40:53,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742442_1618 (size=14794) 2024-11-13T10:40:53,241 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:53,244 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130ad1e3abc7824ea4bd0fcab5f6a7c39e_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130ad1e3abc7824ea4bd0fcab5f6a7c39e_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:53,245 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/8260a34745e641f4a83080282198cb2e, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:53,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/8260a34745e641f4a83080282198cb2e is 175, key is test_row_0/A:col10/1731494451066/Put/seqid=0 2024-11-13T10:40:53,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742443_1619 (size=39749) 2024-11-13T10:40:53,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494513236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,265 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 770665a7984d,39101,1731494292956 for 'test_row_0' in c32615157847c6ad00128cef38e49033 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T10:40:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494513265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494513265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494513265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494513377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494513377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494513378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494513587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494513587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494513587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/8260a34745e641f4a83080282198cb2e 2024-11-13T10:40:53,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/922c176809a841a789e4dd91f23f65a5 is 50, key is test_row_0/B:col10/1731494451066/Put/seqid=0 2024-11-13T10:40:53,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742444_1620 (size=12151) 2024-11-13T10:40:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T10:40:53,797 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:53,798 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-13T10:40:53,800 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:53,802 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:53,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T10:40:53,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494513897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494513898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T10:40:53,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494513908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:53,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:53,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T10:40:53,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:53,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:53,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:53,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:53,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:54,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/922c176809a841a789e4dd91f23f65a5 2024-11-13T10:40:54,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/b7614fa92d414d879923e3da4856c7ac is 50, key is test_row_0/C:col10/1731494451066/Put/seqid=0 2024-11-13T10:40:54,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742445_1621 (size=12151) 2024-11-13T10:40:54,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T10:40:54,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:54,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:54,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:54,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:54,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T10:40:54,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T10:40:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:54,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:54,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494514408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:54,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494514416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:54,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T10:40:54,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:54,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:54,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:54,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:54,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:54,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494514420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:54,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T10:40:54,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/b7614fa92d414d879923e3da4856c7ac 2024-11-13T10:40:54,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/8260a34745e641f4a83080282198cb2e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/8260a34745e641f4a83080282198cb2e 2024-11-13T10:40:54,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/8260a34745e641f4a83080282198cb2e, entries=200, sequenceid=202, filesize=38.8 K 2024-11-13T10:40:54,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/922c176809a841a789e4dd91f23f65a5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/922c176809a841a789e4dd91f23f65a5 2024-11-13T10:40:54,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/922c176809a841a789e4dd91f23f65a5, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T10:40:54,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/b7614fa92d414d879923e3da4856c7ac as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b7614fa92d414d879923e3da4856c7ac 2024-11-13T10:40:54,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b7614fa92d414d879923e3da4856c7ac, entries=150, sequenceid=202, filesize=11.9 K 2024-11-13T10:40:54,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for c32615157847c6ad00128cef38e49033 in 1311ms, sequenceid=202, compaction requested=true 2024-11-13T10:40:54,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:54,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:54,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:54,534 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:54,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:54,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:54,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:54,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:54,534 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:54,535 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:54,535 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:54,535 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/A is initiating minor compaction (all files) 2024-11-13T10:40:54,535 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/B is initiating minor compaction (all files) 2024-11-13T10:40:54,536 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/A in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:54,536 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/B in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:54,536 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/a839e5f08bf34be58a6d7c156f23ef7a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/daefd0d0cb7f4f11b774da94383f41cc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/8260a34745e641f4a83080282198cb2e] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=91.5 K 2024-11-13T10:40:54,536 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/99a3a58b9ee14df2bd0cd1904457018e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e12594e21f9469a97827791cbed6378, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/922c176809a841a789e4dd91f23f65a5] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=33.6 K 2024-11-13T10:40:54,536 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:54,536 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/a839e5f08bf34be58a6d7c156f23ef7a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/daefd0d0cb7f4f11b774da94383f41cc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/8260a34745e641f4a83080282198cb2e] 2024-11-13T10:40:54,536 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 99a3a58b9ee14df2bd0cd1904457018e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494447627 2024-11-13T10:40:54,536 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting a839e5f08bf34be58a6d7c156f23ef7a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494447627 2024-11-13T10:40:54,536 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting daefd0d0cb7f4f11b774da94383f41cc, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731494449791 2024-11-13T10:40:54,536 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e12594e21f9469a97827791cbed6378, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731494449791 2024-11-13T10:40:54,537 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8260a34745e641f4a83080282198cb2e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731494451063 2024-11-13T10:40:54,537 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 922c176809a841a789e4dd91f23f65a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731494451063 2024-11-13T10:40:54,565 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:54,566 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#B#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:54,567 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/45e4edb21ec64230b12ce17787263967 is 50, key is test_row_0/B:col10/1731494451066/Put/seqid=0 2024-11-13T10:40:54,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T10:40:54,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:54,574 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-13T10:40:54,584 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411138aa395d4a4004e478ac340fb90cc6312_c32615157847c6ad00128cef38e49033 store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:54,586 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411138aa395d4a4004e478ac340fb90cc6312_c32615157847c6ad00128cef38e49033, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:54,586 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138aa395d4a4004e478ac340fb90cc6312_c32615157847c6ad00128cef38e49033 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:54,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742446_1622 (size=12595) 2024-11-13T10:40:54,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c8e41cb508fc4e7484aed3a313635df4_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494453231/Put/seqid=0 2024-11-13T10:40:54,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742447_1623 (size=4469) 2024-11-13T10:40:54,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742448_1624 (size=12304) 2024-11-13T10:40:54,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:54,648 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c8e41cb508fc4e7484aed3a313635df4_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c8e41cb508fc4e7484aed3a313635df4_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:54,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/eb6c0af27bff451c83d48e17dfa53f1a, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:54,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/eb6c0af27bff451c83d48e17dfa53f1a is 175, key is test_row_0/A:col10/1731494453231/Put/seqid=0 2024-11-13T10:40:54,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742449_1625 (size=31105) 2024-11-13T10:40:54,654 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/eb6c0af27bff451c83d48e17dfa53f1a 2024-11-13T10:40:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/a23ccb7ad45c4764b8795b2c3a35f8e5 is 50, key is test_row_0/B:col10/1731494453231/Put/seqid=0 2024-11-13T10:40:54,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742450_1626 (size=12151) 2024-11-13T10:40:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T10:40:55,024 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/45e4edb21ec64230b12ce17787263967 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/45e4edb21ec64230b12ce17787263967 2024-11-13T10:40:55,029 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/B of c32615157847c6ad00128cef38e49033 into 45e4edb21ec64230b12ce17787263967(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:55,029 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:55,029 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/B, priority=13, startTime=1731494454534; duration=0sec 2024-11-13T10:40:55,029 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:55,029 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:B 2024-11-13T10:40:55,029 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:55,030 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:55,030 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/C is initiating minor compaction (all files) 2024-11-13T10:40:55,030 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/C in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:55,030 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c0672ad649c44a76a30875c538154c09, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/bd3328d2e57f43a88b06cef0d649a14c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b7614fa92d414d879923e3da4856c7ac] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=33.6 K 2024-11-13T10:40:55,031 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting c0672ad649c44a76a30875c538154c09, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731494447627 2024-11-13T10:40:55,031 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3328d2e57f43a88b06cef0d649a14c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731494449791 2024-11-13T10:40:55,031 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting b7614fa92d414d879923e3da4856c7ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731494451063 2024-11-13T10:40:55,041 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#C#compaction#529 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:55,042 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/c65ede6ee42344fab06dd197423e5917 is 50, key is test_row_0/C:col10/1731494451066/Put/seqid=0 2024-11-13T10:40:55,043 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#A#compaction#526 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:55,044 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/1447de1120de448fb8837e1753d1cc27 is 175, key is test_row_0/A:col10/1731494451066/Put/seqid=0 2024-11-13T10:40:55,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742451_1627 (size=12595) 2024-11-13T10:40:55,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742452_1628 (size=31549) 2024-11-13T10:40:55,058 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/1447de1120de448fb8837e1753d1cc27 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1447de1120de448fb8837e1753d1cc27 2024-11-13T10:40:55,062 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/A of c32615157847c6ad00128cef38e49033 into 1447de1120de448fb8837e1753d1cc27(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:55,062 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:55,062 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/A, priority=13, startTime=1731494454534; duration=0sec 2024-11-13T10:40:55,063 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:55,063 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:A 2024-11-13T10:40:55,064 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/a23ccb7ad45c4764b8795b2c3a35f8e5 2024-11-13T10:40:55,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/6b0770f4f4aa4986be2d55f7e155d71d is 50, key is test_row_0/C:col10/1731494453231/Put/seqid=0 2024-11-13T10:40:55,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742453_1629 (size=12151) 2024-11-13T10:40:55,073 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/6b0770f4f4aa4986be2d55f7e155d71d 2024-11-13T10:40:55,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/eb6c0af27bff451c83d48e17dfa53f1a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/eb6c0af27bff451c83d48e17dfa53f1a 2024-11-13T10:40:55,080 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/eb6c0af27bff451c83d48e17dfa53f1a, entries=150, sequenceid=209, filesize=30.4 K 2024-11-13T10:40:55,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/a23ccb7ad45c4764b8795b2c3a35f8e5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a23ccb7ad45c4764b8795b2c3a35f8e5 2024-11-13T10:40:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,084 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a23ccb7ad45c4764b8795b2c3a35f8e5, entries=150, sequenceid=209, filesize=11.9 K 2024-11-13T10:40:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/6b0770f4f4aa4986be2d55f7e155d71d as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6b0770f4f4aa4986be2d55f7e155d71d 2024-11-13T10:40:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,089 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6b0770f4f4aa4986be2d55f7e155d71d, entries=150, sequenceid=209, filesize=11.9 K 2024-11-13T10:40:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,093 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for c32615157847c6ad00128cef38e49033 in 519ms, sequenceid=209, compaction requested=false 2024-11-13T10:40:55,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:55,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-13T10:40:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-13T10:40:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-13T10:40:55,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2920 sec 2024-11-13T10:40:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.2990 sec 2024-11-13T10:40:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:55,378 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:55,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:55,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136d380e70d64248a790cff8e594b21469_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494455370/Put/seqid=0 2024-11-13T10:40:55,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742454_1630 (size=12304) 2024-11-13T10:40:55,391 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:55,395 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136d380e70d64248a790cff8e594b21469_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136d380e70d64248a790cff8e594b21469_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:55,397 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/d488fbce69a74759b5c5ed6bf8b7681c, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:55,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/d488fbce69a74759b5c5ed6bf8b7681c is 175, key is test_row_0/A:col10/1731494455370/Put/seqid=0 2024-11-13T10:40:55,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742455_1631 (size=31105) 2024-11-13T10:40:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494515449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494515450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,456 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/c65ede6ee42344fab06dd197423e5917 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c65ede6ee42344fab06dd197423e5917 2024-11-13T10:40:55,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494515453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494515454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,460 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/C of c32615157847c6ad00128cef38e49033 into c65ede6ee42344fab06dd197423e5917(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:55,460 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:55,460 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/C, priority=13, startTime=1731494454534; duration=0sec 2024-11-13T10:40:55,461 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:55,461 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:C 2024-11-13T10:40:55,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494515556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494515556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494515567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494515567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494515767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494515767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494515776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494515786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:55,813 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/d488fbce69a74759b5c5ed6bf8b7681c 2024-11-13T10:40:55,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/64ce529054df4347b7a8e156dbffe555 is 50, key is test_row_0/B:col10/1731494455370/Put/seqid=0 2024-11-13T10:40:55,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742456_1632 (size=12151) 2024-11-13T10:40:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T10:40:55,947 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:40:55,947 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:40:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-13T10:40:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T10:40:55,949 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:40:55,950 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:40:55,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:40:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T10:40:56,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494516077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494516077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494516089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494516097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:56,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:56,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/64ce529054df4347b7a8e156dbffe555 2024-11-13T10:40:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T10:40:56,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/dc7dbcd9dca142a9aba32b8f00a61d51 is 50, key is test_row_0/C:col10/1731494455370/Put/seqid=0 2024-11-13T10:40:56,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742457_1633 (size=12151) 2024-11-13T10:40:56,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/dc7dbcd9dca142a9aba32b8f00a61d51 2024-11-13T10:40:56,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:56,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:56,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/d488fbce69a74759b5c5ed6bf8b7681c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/d488fbce69a74759b5c5ed6bf8b7681c 2024-11-13T10:40:56,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/d488fbce69a74759b5c5ed6bf8b7681c, entries=150, sequenceid=222, filesize=30.4 K 2024-11-13T10:40:56,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/64ce529054df4347b7a8e156dbffe555 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/64ce529054df4347b7a8e156dbffe555 2024-11-13T10:40:56,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/64ce529054df4347b7a8e156dbffe555, entries=150, sequenceid=222, filesize=11.9 K 2024-11-13T10:40:56,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/dc7dbcd9dca142a9aba32b8f00a61d51 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/dc7dbcd9dca142a9aba32b8f00a61d51 2024-11-13T10:40:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T10:40:56,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/dc7dbcd9dca142a9aba32b8f00a61d51, entries=150, sequenceid=222, filesize=11.9 K 2024-11-13T10:40:56,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for c32615157847c6ad00128cef38e49033 in 1226ms, sequenceid=222, compaction requested=true 2024-11-13T10:40:56,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:56,605 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:40:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:40:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:40:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:40:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:56,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T10:40:56,608 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:56,612 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:56,613 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/A is initiating minor compaction (all files) 2024-11-13T10:40:56,613 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/A in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,613 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1447de1120de448fb8837e1753d1cc27, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/eb6c0af27bff451c83d48e17dfa53f1a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/d488fbce69a74759b5c5ed6bf8b7681c] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=91.6 K 2024-11-13T10:40:56,613 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,613 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1447de1120de448fb8837e1753d1cc27, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/eb6c0af27bff451c83d48e17dfa53f1a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/d488fbce69a74759b5c5ed6bf8b7681c] 2024-11-13T10:40:56,619 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1447de1120de448fb8837e1753d1cc27, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731494451063 2024-11-13T10:40:56,619 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:56,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:56,619 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/B is initiating minor compaction (all files) 2024-11-13T10:40:56,619 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/B in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,619 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/45e4edb21ec64230b12ce17787263967, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a23ccb7ad45c4764b8795b2c3a35f8e5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/64ce529054df4347b7a8e156dbffe555] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=36.0 K 2024-11-13T10:40:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,629 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb6c0af27bff451c83d48e17dfa53f1a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731494453230 2024-11-13T10:40:56,629 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 45e4edb21ec64230b12ce17787263967, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731494451063 2024-11-13T10:40:56,640 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting d488fbce69a74759b5c5ed6bf8b7681c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731494455370 2024-11-13T10:40:56,641 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting a23ccb7ad45c4764b8795b2c3a35f8e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731494453230 2024-11-13T10:40:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494516635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494516635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494516637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,649 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ce529054df4347b7a8e156dbffe555, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731494455370 2024-11-13T10:40:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494516646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130b0db71dbe064fe39b6f8ea5922e0095_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494455452/Put/seqid=0 2024-11-13T10:40:56,696 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#B#compaction#535 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:56,696 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/aaf7a9e10ab1422497bfcef041f974f1 is 50, key is test_row_0/B:col10/1731494455370/Put/seqid=0 2024-11-13T10:40:56,707 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:56,753 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111379f419c6811e46b5a26cfb2eec069075_c32615157847c6ad00128cef38e49033 store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:56,755 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111379f419c6811e46b5a26cfb2eec069075_c32615157847c6ad00128cef38e49033, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:56,755 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111379f419c6811e46b5a26cfb2eec069075_c32615157847c6ad00128cef38e49033 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:56,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742459_1635 (size=12697) 2024-11-13T10:40:56,765 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/aaf7a9e10ab1422497bfcef041f974f1 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/aaf7a9e10ab1422497bfcef041f974f1 2024-11-13T10:40:56,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494516760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494516764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494516764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,771 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/B of c32615157847c6ad00128cef38e49033 into aaf7a9e10ab1422497bfcef041f974f1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:56,771 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:56,772 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/B, priority=13, startTime=1731494456605; duration=0sec 2024-11-13T10:40:56,772 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:40:56,772 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:B 2024-11-13T10:40:56,772 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T10:40:56,773 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T10:40:56,773 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/C is initiating minor compaction (all files) 2024-11-13T10:40:56,773 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/C in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,773 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c65ede6ee42344fab06dd197423e5917, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6b0770f4f4aa4986be2d55f7e155d71d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/dc7dbcd9dca142a9aba32b8f00a61d51] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=36.0 K 2024-11-13T10:40:56,774 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting c65ede6ee42344fab06dd197423e5917, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731494451063 2024-11-13T10:40:56,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,775 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0770f4f4aa4986be2d55f7e155d71d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731494453230 2024-11-13T10:40:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494516767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,775 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting dc7dbcd9dca142a9aba32b8f00a61d51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731494455370 2024-11-13T10:40:56,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742458_1634 (size=12304) 2024-11-13T10:40:56,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:56,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:56,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,794 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#C#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:56,795 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/336d4b28f67f45d299b6d8bc4f5217f5 is 50, key is test_row_0/C:col10/1731494455370/Put/seqid=0 2024-11-13T10:40:56,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:56,801 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130b0db71dbe064fe39b6f8ea5922e0095_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130b0db71dbe064fe39b6f8ea5922e0095_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:56,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/558cae945e114e30b11288b2a3712b51, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:56,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/558cae945e114e30b11288b2a3712b51 is 175, key is test_row_0/A:col10/1731494455452/Put/seqid=0 2024-11-13T10:40:56,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742460_1636 (size=4469) 2024-11-13T10:40:56,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742461_1637 (size=12697) 2024-11-13T10:40:56,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742462_1638 (size=31105) 2024-11-13T10:40:56,846 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/336d4b28f67f45d299b6d8bc4f5217f5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/336d4b28f67f45d299b6d8bc4f5217f5 2024-11-13T10:40:56,891 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/C of c32615157847c6ad00128cef38e49033 into 336d4b28f67f45d299b6d8bc4f5217f5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:56,891 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:56,891 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/C, priority=13, startTime=1731494456605; duration=0sec 2024-11-13T10:40:56,892 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:56,892 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:C 2024-11-13T10:40:56,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:56,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:56,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494516977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494516977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494516977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:56,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494516981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T10:40:57,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:57,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,212 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#A#compaction#536 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:40:57,213 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/6a45bbd9f9834d0b91cae9fcf1c8b593 is 175, key is test_row_0/A:col10/1731494455370/Put/seqid=0 2024-11-13T10:40:57,245 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/558cae945e114e30b11288b2a3712b51 2024-11-13T10:40:57,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:57,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:57,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742463_1639 (size=31651) 2024-11-13T10:40:57,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/2e931fc735ee4bed983e347c3d2cde15 is 50, key is test_row_0/B:col10/1731494455452/Put/seqid=0 2024-11-13T10:40:57,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494517287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:57,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494517287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:57,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494517288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:57,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494517297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:57,313 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/6a45bbd9f9834d0b91cae9fcf1c8b593 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6a45bbd9f9834d0b91cae9fcf1c8b593 2024-11-13T10:40:57,321 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c32615157847c6ad00128cef38e49033/A of c32615157847c6ad00128cef38e49033 into 6a45bbd9f9834d0b91cae9fcf1c8b593(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:40:57,321 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:57,321 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/A, priority=13, startTime=1731494456605; duration=0sec 2024-11-13T10:40:57,321 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:40:57,321 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:A 2024-11-13T10:40:57,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742464_1640 (size=12151) 2024-11-13T10:40:57,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:57,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:57,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:57,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:57,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/2e931fc735ee4bed983e347c3d2cde15 2024-11-13T10:40:57,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:57,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:57,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/91fe5a22a51d4dd2b305a735c44e07b2 is 50, key is test_row_0/C:col10/1731494455452/Put/seqid=0 2024-11-13T10:40:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494517797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494517797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:57,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494517807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:57,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494517817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:57,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:57,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742465_1641 (size=12151) 2024-11-13T10:40:57,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/91fe5a22a51d4dd2b305a735c44e07b2 2024-11-13T10:40:57,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/558cae945e114e30b11288b2a3712b51 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/558cae945e114e30b11288b2a3712b51 2024-11-13T10:40:57,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:57,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:57,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:57,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:40:57,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/558cae945e114e30b11288b2a3712b51, entries=150, sequenceid=250, filesize=30.4 K 2024-11-13T10:40:57,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/2e931fc735ee4bed983e347c3d2cde15 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e931fc735ee4bed983e347c3d2cde15 2024-11-13T10:40:57,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e931fc735ee4bed983e347c3d2cde15, entries=150, sequenceid=250, filesize=11.9 K 2024-11-13T10:40:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/91fe5a22a51d4dd2b305a735c44e07b2 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/91fe5a22a51d4dd2b305a735c44e07b2 2024-11-13T10:40:57,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/91fe5a22a51d4dd2b305a735c44e07b2, entries=150, sequenceid=250, filesize=11.9 K 2024-11-13T10:40:57,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for c32615157847c6ad00128cef38e49033 in 1391ms, sequenceid=250, compaction requested=false 2024-11-13T10:40:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:58,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-13T10:40:58,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:58,076 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:40:58,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T10:40:58,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e5188aa3084e436cbdd45200b456b0d5_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494456632/Put/seqid=0 2024-11-13T10:40:58,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742466_1642 (size=12404) 2024-11-13T10:40:58,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:58,639 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e5188aa3084e436cbdd45200b456b0d5_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e5188aa3084e436cbdd45200b456b0d5_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:58,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/f19f5daf313f4887acf83c3eb279972c, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:58,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/f19f5daf313f4887acf83c3eb279972c is 175, key is test_row_0/A:col10/1731494456632/Put/seqid=0 2024-11-13T10:40:58,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742467_1643 (size=31205) 2024-11-13T10:40:58,690 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/f19f5daf313f4887acf83c3eb279972c 2024-11-13T10:40:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/498e2cceb77144c2bd20f21557b56310 is 50, key is test_row_0/B:col10/1731494456632/Put/seqid=0 2024-11-13T10:40:58,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742468_1644 (size=12251) 2024-11-13T10:40:58,820 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/498e2cceb77144c2bd20f21557b56310 2024-11-13T10:40:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:58,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:40:58,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/04ff877040164007adb1d064932dab8c is 50, key is test_row_0/C:col10/1731494456632/Put/seqid=0 2024-11-13T10:40:58,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742469_1645 (size=12251) 2024-11-13T10:40:58,897 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/04ff877040164007adb1d064932dab8c 2024-11-13T10:40:58,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/f19f5daf313f4887acf83c3eb279972c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f19f5daf313f4887acf83c3eb279972c 2024-11-13T10:40:58,928 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f19f5daf313f4887acf83c3eb279972c, entries=150, sequenceid=264, filesize=30.5 K 2024-11-13T10:40:58,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/498e2cceb77144c2bd20f21557b56310 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/498e2cceb77144c2bd20f21557b56310 2024-11-13T10:40:58,947 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/498e2cceb77144c2bd20f21557b56310, entries=150, sequenceid=264, filesize=12.0 K 2024-11-13T10:40:58,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494518940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:58,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:58,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494518943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:58,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:58,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:58,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:58,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494518945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:58,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:58,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494518947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:58,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/04ff877040164007adb1d064932dab8c as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/04ff877040164007adb1d064932dab8c 2024-11-13T10:40:58,983 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/04ff877040164007adb1d064932dab8c, entries=150, sequenceid=264, filesize=12.0 K 2024-11-13T10:40:58,984 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for c32615157847c6ad00128cef38e49033 in 908ms, sequenceid=264, compaction requested=true 2024-11-13T10:40:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:40:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:40:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-13T10:40:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-13T10:40:58,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-13T10:40:58,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0470 sec 2024-11-13T10:40:59,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 3.0520 sec 2024-11-13T10:40:59,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(8855): Flush requested on c32615157847c6ad00128cef38e49033 2024-11-13T10:40:59,062 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T10:40:59,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T10:40:59,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130cb091fa10ac4ecf9973ca6e6ceed243_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494458943/Put/seqid=0 2024-11-13T10:40:59,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494519089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494519095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494519101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494519104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742470_1646 (size=12454) 2024-11-13T10:40:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494519207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494519207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494519207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494519228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494519420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494519427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494519427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494519437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,542 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:40:59,581 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130cb091fa10ac4ecf9973ca6e6ceed243_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130cb091fa10ac4ecf9973ca6e6ceed243_c32615157847c6ad00128cef38e49033 2024-11-13T10:40:59,595 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/6359ccb069db46169e819ff0df96a0f8, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:40:59,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/6359ccb069db46169e819ff0df96a0f8 is 175, key is test_row_0/A:col10/1731494458943/Put/seqid=0 2024-11-13T10:40:59,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742471_1647 (size=31255) 2024-11-13T10:40:59,641 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/6359ccb069db46169e819ff0df96a0f8 2024-11-13T10:40:59,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/17a8f9be258642658ae165cad17d043a is 50, key is test_row_0/B:col10/1731494458943/Put/seqid=0 2024-11-13T10:40:59,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742472_1648 (size=12301) 2024-11-13T10:40:59,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494519741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494519744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494519749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:40:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494519749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:40:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:40:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:40:59,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:41:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T10:41:00,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/17a8f9be258642658ae165cad17d043a 2024-11-13T10:41:00,140 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:41:00,141 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:41:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-13T10:41:00,143 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:41:00,144 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:41:00,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:41:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T10:41:00,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/fdb2954a1784424dac40d1c4901d7615 is 50, key is test_row_0/C:col10/1731494458943/Put/seqid=0 2024-11-13T10:41:00,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742473_1649 (size=12301) 2024-11-13T10:41:00,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/fdb2954a1784424dac40d1c4901d7615 2024-11-13T10:41:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T10:41:00,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:41:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45514 deadline: 1731494520261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:41:00,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:41:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45498 deadline: 1731494520261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:41:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:41:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:41:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:41:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:41:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:41:00,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:41:00,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:41:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731494520268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:41:00,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T10:41:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45556 deadline: 1731494520268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 2024-11-13T10:41:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:41:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:41:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:41:00,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 , the old value is region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:41:00,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T10:41:00,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., hostname=770665a7984d,39101,1731494292956, seqNum=5 because the exception is null or not the one we care about 2024-11-13T10:41:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/6359ccb069db46169e819ff0df96a0f8 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6359ccb069db46169e819ff0df96a0f8 2024-11-13T10:41:00,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T10:41:00,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:00,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. as already flushing 2024-11-13T10:41:00,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:00,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:41:00,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:41:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T10:41:00,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6359ccb069db46169e819ff0df96a0f8, entries=150, sequenceid=291, filesize=30.5 K 2024-11-13T10:41:00,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/17a8f9be258642658ae165cad17d043a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/17a8f9be258642658ae165cad17d043a 2024-11-13T10:41:00,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/17a8f9be258642658ae165cad17d043a, entries=150, sequenceid=291, filesize=12.0 K 2024-11-13T10:41:00,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/fdb2954a1784424dac40d1c4901d7615 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/fdb2954a1784424dac40d1c4901d7615 2024-11-13T10:41:00,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/fdb2954a1784424dac40d1c4901d7615, entries=150, sequenceid=291, filesize=12.0 K 2024-11-13T10:41:00,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for c32615157847c6ad00128cef38e49033 in 1312ms, sequenceid=291, compaction requested=true 2024-11-13T10:41:00,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:41:00,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T10:41:00,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:41:00,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T10:41:00,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T10:41:00,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c32615157847c6ad00128cef38e49033:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T10:41:00,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T10:41:00,375 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:41:00,376 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:41:00,388 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125216 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:41:00,388 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/A is initiating minor compaction (all files) 2024-11-13T10:41:00,388 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/A in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:00,388 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6a45bbd9f9834d0b91cae9fcf1c8b593, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/558cae945e114e30b11288b2a3712b51, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f19f5daf313f4887acf83c3eb279972c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6359ccb069db46169e819ff0df96a0f8] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=122.3 K 2024-11-13T10:41:00,388 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:00,389 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. files: [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6a45bbd9f9834d0b91cae9fcf1c8b593, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/558cae945e114e30b11288b2a3712b51, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f19f5daf313f4887acf83c3eb279972c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6359ccb069db46169e819ff0df96a0f8] 2024-11-13T10:41:00,392 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:41:00,392 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a45bbd9f9834d0b91cae9fcf1c8b593, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731494455370 2024-11-13T10:41:00,392 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/C is initiating minor compaction (all files) 2024-11-13T10:41:00,393 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/C in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:00,393 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/336d4b28f67f45d299b6d8bc4f5217f5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/91fe5a22a51d4dd2b305a735c44e07b2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/04ff877040164007adb1d064932dab8c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/fdb2954a1784424dac40d1c4901d7615] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=48.2 K 2024-11-13T10:41:00,393 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 558cae945e114e30b11288b2a3712b51, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494455444 2024-11-13T10:41:00,393 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting f19f5daf313f4887acf83c3eb279972c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731494456616 2024-11-13T10:41:00,394 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6359ccb069db46169e819ff0df96a0f8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731494458940 2024-11-13T10:41:00,396 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 336d4b28f67f45d299b6d8bc4f5217f5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731494455370 2024-11-13T10:41:00,400 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 91fe5a22a51d4dd2b305a735c44e07b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494455444 2024-11-13T10:41:00,404 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 04ff877040164007adb1d064932dab8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731494456616 2024-11-13T10:41:00,406 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:41:00,412 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting fdb2954a1784424dac40d1c4901d7615, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731494458940 2024-11-13T10:41:00,422 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113c8a9d196062c4c5494ef93f05df7cc2a_c32615157847c6ad00128cef38e49033 store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:41:00,429 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113c8a9d196062c4c5494ef93f05df7cc2a_c32615157847c6ad00128cef38e49033, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:41:00,430 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c8a9d196062c4c5494ef93f05df7cc2a_c32615157847c6ad00128cef38e49033 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:41:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T10:41:00,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-13T10:41:00,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:00,465 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T10:41:00,470 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#C#compaction#547 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:41:00,471 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/62cc32a1693e42569d8f6b423bd982f0 is 50, key is test_row_0/C:col10/1731494458943/Put/seqid=0 2024-11-13T10:41:00,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742474_1650 (size=4469) 2024-11-13T10:41:00,476 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#A#compaction#546 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:41:00,476 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/6f2dc8ca7e824a6bb46d0c93377dbca5 is 175, key is test_row_0/A:col10/1731494458943/Put/seqid=0 2024-11-13T10:41:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111318e198435e8644ffafaa499034506ec7_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494459093/Put/seqid=0 2024-11-13T10:41:00,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742475_1651 (size=12983) 2024-11-13T10:41:00,521 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/62cc32a1693e42569d8f6b423bd982f0 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/62cc32a1693e42569d8f6b423bd982f0 2024-11-13T10:41:00,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742476_1652 (size=31937) 2024-11-13T10:41:00,528 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c32615157847c6ad00128cef38e49033/C of c32615157847c6ad00128cef38e49033 into 62cc32a1693e42569d8f6b423bd982f0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:41:00,528 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:41:00,528 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/C, priority=12, startTime=1731494460375; duration=0sec 2024-11-13T10:41:00,528 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T10:41:00,528 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:C 2024-11-13T10:41:00,528 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T10:41:00,535 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T10:41:00,535 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1541): c32615157847c6ad00128cef38e49033/B is initiating minor compaction (all files) 2024-11-13T10:41:00,535 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c32615157847c6ad00128cef38e49033/B in TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:00,535 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/aaf7a9e10ab1422497bfcef041f974f1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e931fc735ee4bed983e347c3d2cde15, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/498e2cceb77144c2bd20f21557b56310, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/17a8f9be258642658ae165cad17d043a] into tmpdir=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp, totalSize=48.2 K 2024-11-13T10:41:00,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742477_1653 (size=12454) 2024-11-13T10:41:00,537 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting aaf7a9e10ab1422497bfcef041f974f1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731494455370 2024-11-13T10:41:00,542 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e931fc735ee4bed983e347c3d2cde15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731494455444 2024-11-13T10:41:00,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:00,546 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 498e2cceb77144c2bd20f21557b56310, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731494456616 2024-11-13T10:41:00,549 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a8f9be258642658ae165cad17d043a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731494458940 2024-11-13T10:41:00,567 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111318e198435e8644ffafaa499034506ec7_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111318e198435e8644ffafaa499034506ec7_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:00,568 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/6f2dc8ca7e824a6bb46d0c93377dbca5 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6f2dc8ca7e824a6bb46d0c93377dbca5 2024-11-13T10:41:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/f5df1ab51f3148a38b694e8dbadeb4ca, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:41:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/f5df1ab51f3148a38b694e8dbadeb4ca is 175, key is test_row_0/A:col10/1731494459093/Put/seqid=0 2024-11-13T10:41:00,613 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c32615157847c6ad00128cef38e49033#B#compaction#549 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T10:41:00,614 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/b58b0d20888649e791cd08fb8e759487 is 50, key is test_row_0/B:col10/1731494458943/Put/seqid=0 2024-11-13T10:41:00,625 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c32615157847c6ad00128cef38e49033/A of c32615157847c6ad00128cef38e49033 into 6f2dc8ca7e824a6bb46d0c93377dbca5(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:41:00,625 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:41:00,625 INFO [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/A, priority=12, startTime=1731494460375; duration=0sec 2024-11-13T10:41:00,625 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:41:00,625 DEBUG [RS:0;770665a7984d:39101-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:A 2024-11-13T10:41:00,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742479_1655 (size=12983) 2024-11-13T10:41:00,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742478_1654 (size=31255) 2024-11-13T10:41:00,638 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/f5df1ab51f3148a38b694e8dbadeb4ca 2024-11-13T10:41:00,695 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/b58b0d20888649e791cd08fb8e759487 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b58b0d20888649e791cd08fb8e759487 2024-11-13T10:41:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/3127f57e9eb64101a54ef555ac74a375 is 50, key is test_row_0/B:col10/1731494459093/Put/seqid=0 2024-11-13T10:41:00,724 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c32615157847c6ad00128cef38e49033/B of c32615157847c6ad00128cef38e49033 into b58b0d20888649e791cd08fb8e759487(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T10:41:00,724 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:41:00,724 INFO [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., storeName=c32615157847c6ad00128cef38e49033/B, priority=12, startTime=1731494460375; duration=0sec 2024-11-13T10:41:00,724 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T10:41:00,724 DEBUG [RS:0;770665a7984d:39101-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c32615157847c6ad00128cef38e49033:B 2024-11-13T10:41:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T10:41:00,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742480_1656 (size=12301) 2024-11-13T10:41:00,780 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/3127f57e9eb64101a54ef555ac74a375 2024-11-13T10:41:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/536df1949ec14a98b23df2bf8509a304 is 50, key is test_row_0/C:col10/1731494459093/Put/seqid=0 2024-11-13T10:41:00,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742481_1657 (size=12301) 2024-11-13T10:41:00,924 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/536df1949ec14a98b23df2bf8509a304 2024-11-13T10:41:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/f5df1ab51f3148a38b694e8dbadeb4ca as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f5df1ab51f3148a38b694e8dbadeb4ca 2024-11-13T10:41:01,009 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f5df1ab51f3148a38b694e8dbadeb4ca, entries=150, sequenceid=302, filesize=30.5 K 2024-11-13T10:41:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/3127f57e9eb64101a54ef555ac74a375 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/3127f57e9eb64101a54ef555ac74a375 2024-11-13T10:41:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,024 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/3127f57e9eb64101a54ef555ac74a375, entries=150, sequenceid=302, filesize=12.0 K 2024-11-13T10:41:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/536df1949ec14a98b23df2bf8509a304 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/536df1949ec14a98b23df2bf8509a304 2024-11-13T10:41:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,035 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/536df1949ec14a98b23df2bf8509a304, entries=150, sequenceid=302, filesize=12.0 K 2024-11-13T10:41:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,039 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for c32615157847c6ad00128cef38e49033 in 574ms, sequenceid=302, compaction requested=false 2024-11-13T10:41:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:41:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:01,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-13T10:41:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-13T10:41:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-13T10:41:01,050 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 900 msec 2024-11-13T10:41:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 910 msec 2024-11-13T10:41:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-13T10:41:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,280 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:41:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,281 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T10:41:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-13T10:41:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,285 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T10:41:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,286 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T10:41:01,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T10:41:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T10:41:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,329 INFO [Thread-2522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2522. 2024-11-13T10:41:01,329 DEBUG [Thread-2522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:41:01,329 DEBUG [Thread-2522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,329 DEBUG [Thread-2522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:01,330 INFO [Thread-2535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2535. 2024-11-13T10:41:01,330 DEBUG [Thread-2535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:41:01,330 DEBUG [Thread-2535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,330 DEBUG [Thread-2535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,332 INFO [Thread-2520 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2520. 2024-11-13T10:41:01,332 DEBUG [Thread-2520 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:41:01,332 DEBUG [Thread-2520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,332 DEBUG [Thread-2520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,333 INFO [Thread-2528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2528. 2024-11-13T10:41:01,333 DEBUG [Thread-2528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:41:01,334 DEBUG [Thread-2528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,334 DEBUG [Thread-2528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:01,336 INFO [Thread-2524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2524. 2024-11-13T10:41:01,336 DEBUG [Thread-2524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:41:01,337 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,337 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:01,338 INFO [Thread-2531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2531. 2024-11-13T10:41:01,338 DEBUG [Thread-2531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:41:01,338 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,339 DEBUG [Thread-2531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,340 INFO [Thread-2533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2533. 2024-11-13T10:41:01,340 DEBUG [Thread-2533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:41:01,340 DEBUG [Thread-2533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,340 DEBUG [Thread-2533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:01,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,361 INFO [Thread-2539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2539. 2024-11-13T10:41:01,361 DEBUG [Thread-2539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:41:01,361 DEBUG [Thread-2539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,361 DEBUG [Thread-2539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39101 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,392 INFO [Thread-2537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2537. 2024-11-13T10:41:01,392 DEBUG [Thread-2537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:41:01,392 DEBUG [Thread-2537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,392 DEBUG [Thread-2537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T10:41:01,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39101 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-13T10:41:01,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:01,441 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-13T10:41:01,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111308f6f9a4927c45eea76b2b0f76e7c6ed_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494461327/Put/seqid=0 2024-11-13T10:41:01,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742482_1658 (size=12454) 2024-11-13T10:41:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T10:41:01,769 INFO [Thread-2526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2526. 2024-11-13T10:41:01,769 DEBUG [Thread-2526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T10:41:01,769 DEBUG [Thread-2526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,770 DEBUG [Thread-2526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:01,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:01,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:01,909 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111308f6f9a4927c45eea76b2b0f76e7c6ed_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308f6f9a4927c45eea76b2b0f76e7c6ed_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:01,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/ac0041c02cea44afa318bdca656e3830, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:41:01,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/ac0041c02cea44afa318bdca656e3830 is 175, key is test_row_0/A:col10/1731494461327/Put/seqid=0 2024-11-13T10:41:01,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742483_1659 (size=31255) 2024-11-13T10:41:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T10:41:01,918 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/ac0041c02cea44afa318bdca656e3830 2024-11-13T10:41:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/b824704b5429435c9ab3b0ddd2c243df is 50, key is test_row_0/B:col10/1731494461327/Put/seqid=0 2024-11-13T10:41:01,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742484_1660 (size=12301) 2024-11-13T10:41:02,330 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/b824704b5429435c9ab3b0ddd2c243df 2024-11-13T10:41:02,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/9073ccfd8eff495692ce3b16c9ec165e is 50, key is test_row_0/C:col10/1731494461327/Put/seqid=0 2024-11-13T10:41:02,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742485_1661 (size=12301) 2024-11-13T10:41:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T10:41:02,743 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/9073ccfd8eff495692ce3b16c9ec165e 2024-11-13T10:41:02,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/ac0041c02cea44afa318bdca656e3830 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/ac0041c02cea44afa318bdca656e3830 2024-11-13T10:41:02,751 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/ac0041c02cea44afa318bdca656e3830, entries=150, sequenceid=312, filesize=30.5 K 2024-11-13T10:41:02,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/b824704b5429435c9ab3b0ddd2c243df as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b824704b5429435c9ab3b0ddd2c243df 2024-11-13T10:41:02,757 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b824704b5429435c9ab3b0ddd2c243df, entries=150, sequenceid=312, filesize=12.0 K 2024-11-13T10:41:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/9073ccfd8eff495692ce3b16c9ec165e as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/9073ccfd8eff495692ce3b16c9ec165e 2024-11-13T10:41:02,762 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/9073ccfd8eff495692ce3b16c9ec165e, entries=150, sequenceid=312, filesize=12.0 K 2024-11-13T10:41:02,763 INFO [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=6.71 KB/6870 for c32615157847c6ad00128cef38e49033 in 1322ms, sequenceid=312, compaction requested=true 2024-11-13T10:41:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for c32615157847c6ad00128cef38e49033: 2024-11-13T10:41:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/770665a7984d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-13T10:41:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-13T10:41:02,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-13T10:41:02,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-11-13T10:41:02,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.4860 sec 2024-11-13T10:41:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T10:41:03,448 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2857 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2646 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2739 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2885 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2643 2024-11-13T10:41:03,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T10:41:03,448 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T10:41:03,448 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T10:41:03,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:03,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:03,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:03,449 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T10:41:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T10:41:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T10:41:03,454 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494463453"}]},"ts":"1731494463453"} 2024-11-13T10:41:03,455 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T10:41:03,455 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T10:41:03,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T10:41:03,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, UNASSIGN}] 2024-11-13T10:41:03,459 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, UNASSIGN 2024-11-13T10:41:03,460 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=c32615157847c6ad00128cef38e49033, regionState=CLOSING, regionLocation=770665a7984d,39101,1731494292956 2024-11-13T10:41:03,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, UNASSIGN because future has completed 2024-11-13T10:41:03,462 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T10:41:03,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956}] 2024-11-13T10:41:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T10:41:03,614 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close c32615157847c6ad00128cef38e49033 2024-11-13T10:41:03,614 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T10:41:03,614 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing c32615157847c6ad00128cef38e49033, disabling compactions & flushes 2024-11-13T10:41:03,614 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:03,614 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:03,614 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. after waiting 0 ms 2024-11-13T10:41:03,614 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:03,614 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing c32615157847c6ad00128cef38e49033 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-13T10:41:03,620 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f7a7327fd3274155bd07b3354df8a560_c32615157847c6ad00128cef38e49033 is 50, key is test_row_0/A:col10/1731494461767/Put/seqid=0 2024-11-13T10:41:03,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742486_1662 (size=7374) 2024-11-13T10:41:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T10:41:04,031 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T10:41:04,034 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f7a7327fd3274155bd07b3354df8a560_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f7a7327fd3274155bd07b3354df8a560_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:04,035 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/c9556a871d00474492cc72ef52a51a1a, store: [table=TestAcidGuarantees family=A region=c32615157847c6ad00128cef38e49033] 2024-11-13T10:41:04,036 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/c9556a871d00474492cc72ef52a51a1a is 175, key is test_row_0/A:col10/1731494461767/Put/seqid=0 2024-11-13T10:41:04,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742487_1663 (size=13865) 2024-11-13T10:41:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T10:41:04,463 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/c9556a871d00474492cc72ef52a51a1a 2024-11-13T10:41:04,469 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/6aaa72ad7f3745a1a7962c095b893623 is 50, key is test_row_0/B:col10/1731494461767/Put/seqid=0 2024-11-13T10:41:04,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742488_1664 (size=7415) 2024-11-13T10:41:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T10:41:04,875 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/6aaa72ad7f3745a1a7962c095b893623 2024-11-13T10:41:04,882 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/8d2dcfbf602e4e7baf61772e65095b19 is 50, key is test_row_0/C:col10/1731494461767/Put/seqid=0 2024-11-13T10:41:04,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742489_1665 (size=7415) 2024-11-13T10:41:05,294 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/8d2dcfbf602e4e7baf61772e65095b19 2024-11-13T10:41:05,297 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/A/c9556a871d00474492cc72ef52a51a1a as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/c9556a871d00474492cc72ef52a51a1a 2024-11-13T10:41:05,299 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/c9556a871d00474492cc72ef52a51a1a, entries=50, sequenceid=316, filesize=13.5 K 2024-11-13T10:41:05,300 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/B/6aaa72ad7f3745a1a7962c095b893623 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/6aaa72ad7f3745a1a7962c095b893623 2024-11-13T10:41:05,302 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/6aaa72ad7f3745a1a7962c095b893623, entries=50, sequenceid=316, filesize=7.2 K 2024-11-13T10:41:05,303 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/.tmp/C/8d2dcfbf602e4e7baf61772e65095b19 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/8d2dcfbf602e4e7baf61772e65095b19 2024-11-13T10:41:05,305 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/8d2dcfbf602e4e7baf61772e65095b19, entries=50, sequenceid=316, filesize=7.2 K 2024-11-13T10:41:05,305 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for c32615157847c6ad00128cef38e49033 in 1691ms, sequenceid=316, compaction requested=true 2024-11-13T10:41:05,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/e008dc1d241441f18da4d425c1adc1ca, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/b66af2fe76624f7f83449f592ead6bf8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/077afe01918142deadfa5f10efc7c7a0, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/105ecc8c40484fcebe0c0395d9415774, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/44dbd5a9d945467385465f5a21b04f67, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/946db4be148a4c0da65a6c2b448ea3d3, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/dcc169d9e192417fbf758825d8879e8f, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1e8a3167a69a4132a965f797151a6daf, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/5ba20a20a88d4cd2a449da52c50e2100, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/0f02b344ad6349d9891c9e5264296bc6, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/a839e5f08bf34be58a6d7c156f23ef7a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/daefd0d0cb7f4f11b774da94383f41cc, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/8260a34745e641f4a83080282198cb2e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1447de1120de448fb8837e1753d1cc27, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/eb6c0af27bff451c83d48e17dfa53f1a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6a45bbd9f9834d0b91cae9fcf1c8b593, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/d488fbce69a74759b5c5ed6bf8b7681c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/558cae945e114e30b11288b2a3712b51, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f19f5daf313f4887acf83c3eb279972c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6359ccb069db46169e819ff0df96a0f8] to archive 2024-11-13T10:41:05,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:41:05,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/e008dc1d241441f18da4d425c1adc1ca to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/e008dc1d241441f18da4d425c1adc1ca 2024-11-13T10:41:05,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/b66af2fe76624f7f83449f592ead6bf8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/b66af2fe76624f7f83449f592ead6bf8 2024-11-13T10:41:05,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/077afe01918142deadfa5f10efc7c7a0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/077afe01918142deadfa5f10efc7c7a0 2024-11-13T10:41:05,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/105ecc8c40484fcebe0c0395d9415774 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/105ecc8c40484fcebe0c0395d9415774 2024-11-13T10:41:05,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/44dbd5a9d945467385465f5a21b04f67 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/44dbd5a9d945467385465f5a21b04f67 2024-11-13T10:41:05,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/946db4be148a4c0da65a6c2b448ea3d3 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/946db4be148a4c0da65a6c2b448ea3d3 2024-11-13T10:41:05,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/dcc169d9e192417fbf758825d8879e8f to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/dcc169d9e192417fbf758825d8879e8f 2024-11-13T10:41:05,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1e8a3167a69a4132a965f797151a6daf to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1e8a3167a69a4132a965f797151a6daf 2024-11-13T10:41:05,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/5ba20a20a88d4cd2a449da52c50e2100 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/5ba20a20a88d4cd2a449da52c50e2100 2024-11-13T10:41:05,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/0f02b344ad6349d9891c9e5264296bc6 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/0f02b344ad6349d9891c9e5264296bc6 2024-11-13T10:41:05,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/a839e5f08bf34be58a6d7c156f23ef7a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/a839e5f08bf34be58a6d7c156f23ef7a 2024-11-13T10:41:05,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/daefd0d0cb7f4f11b774da94383f41cc to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/daefd0d0cb7f4f11b774da94383f41cc 2024-11-13T10:41:05,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/8260a34745e641f4a83080282198cb2e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/8260a34745e641f4a83080282198cb2e 2024-11-13T10:41:05,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1447de1120de448fb8837e1753d1cc27 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/1447de1120de448fb8837e1753d1cc27 2024-11-13T10:41:05,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/eb6c0af27bff451c83d48e17dfa53f1a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/eb6c0af27bff451c83d48e17dfa53f1a 2024-11-13T10:41:05,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6a45bbd9f9834d0b91cae9fcf1c8b593 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6a45bbd9f9834d0b91cae9fcf1c8b593 2024-11-13T10:41:05,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/d488fbce69a74759b5c5ed6bf8b7681c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/d488fbce69a74759b5c5ed6bf8b7681c 2024-11-13T10:41:05,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/558cae945e114e30b11288b2a3712b51 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/558cae945e114e30b11288b2a3712b51 2024-11-13T10:41:05,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f19f5daf313f4887acf83c3eb279972c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f19f5daf313f4887acf83c3eb279972c 2024-11-13T10:41:05,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6359ccb069db46169e819ff0df96a0f8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6359ccb069db46169e819ff0df96a0f8 2024-11-13T10:41:05,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0afa181c2f6245529d2a06b05b2b2b67, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/373164ad097042ac9a5c5760fa8526bd, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/120a95935e704a3a92932504803984a4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/732bd74439cd4afeb42a1c7e5de5d763, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/9b6632afcf41449582b3412e66b97c9d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b372e2c3f80e41c5b7ff9c3439e2268c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e32706acb0c4a19b007d51afba5a461, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a109857961864cdc9d9d66756298b92c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0f75245051b64347aef1f3a375c44162, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/99a3a58b9ee14df2bd0cd1904457018e, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e754925332a4caab78500fcb5751d73, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e12594e21f9469a97827791cbed6378, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/45e4edb21ec64230b12ce17787263967, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/922c176809a841a789e4dd91f23f65a5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a23ccb7ad45c4764b8795b2c3a35f8e5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/aaf7a9e10ab1422497bfcef041f974f1, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/64ce529054df4347b7a8e156dbffe555, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e931fc735ee4bed983e347c3d2cde15, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/498e2cceb77144c2bd20f21557b56310, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/17a8f9be258642658ae165cad17d043a] to archive 2024-11-13T10:41:05,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:41:05,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0afa181c2f6245529d2a06b05b2b2b67 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0afa181c2f6245529d2a06b05b2b2b67 2024-11-13T10:41:05,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/373164ad097042ac9a5c5760fa8526bd to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/373164ad097042ac9a5c5760fa8526bd 2024-11-13T10:41:05,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/120a95935e704a3a92932504803984a4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/120a95935e704a3a92932504803984a4 2024-11-13T10:41:05,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/732bd74439cd4afeb42a1c7e5de5d763 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/732bd74439cd4afeb42a1c7e5de5d763 2024-11-13T10:41:05,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/9b6632afcf41449582b3412e66b97c9d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/9b6632afcf41449582b3412e66b97c9d 2024-11-13T10:41:05,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b372e2c3f80e41c5b7ff9c3439e2268c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b372e2c3f80e41c5b7ff9c3439e2268c 2024-11-13T10:41:05,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e32706acb0c4a19b007d51afba5a461 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e32706acb0c4a19b007d51afba5a461 2024-11-13T10:41:05,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a109857961864cdc9d9d66756298b92c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a109857961864cdc9d9d66756298b92c 2024-11-13T10:41:05,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0f75245051b64347aef1f3a375c44162 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/0f75245051b64347aef1f3a375c44162 2024-11-13T10:41:05,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/99a3a58b9ee14df2bd0cd1904457018e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/99a3a58b9ee14df2bd0cd1904457018e 2024-11-13T10:41:05,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e754925332a4caab78500fcb5751d73 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e754925332a4caab78500fcb5751d73 2024-11-13T10:41:05,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e12594e21f9469a97827791cbed6378 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/5e12594e21f9469a97827791cbed6378 2024-11-13T10:41:05,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/45e4edb21ec64230b12ce17787263967 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/45e4edb21ec64230b12ce17787263967 2024-11-13T10:41:05,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/922c176809a841a789e4dd91f23f65a5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/922c176809a841a789e4dd91f23f65a5 2024-11-13T10:41:05,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a23ccb7ad45c4764b8795b2c3a35f8e5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/a23ccb7ad45c4764b8795b2c3a35f8e5 2024-11-13T10:41:05,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/aaf7a9e10ab1422497bfcef041f974f1 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/aaf7a9e10ab1422497bfcef041f974f1 2024-11-13T10:41:05,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/64ce529054df4347b7a8e156dbffe555 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/64ce529054df4347b7a8e156dbffe555 2024-11-13T10:41:05,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e931fc735ee4bed983e347c3d2cde15 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/2e931fc735ee4bed983e347c3d2cde15 2024-11-13T10:41:05,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/498e2cceb77144c2bd20f21557b56310 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/498e2cceb77144c2bd20f21557b56310 2024-11-13T10:41:05,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/17a8f9be258642658ae165cad17d043a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/17a8f9be258642658ae165cad17d043a 2024-11-13T10:41:05,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/101f86312add43e79d07b5fab3181724, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/d6d45f7d799e4bccaf812f24c522be58, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/3ecc0874c1284833b53fa9378b9f476a, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/e83b736795b5407689af646a48fd39be, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/97ace567baf644cabb91e5fc52df2345, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/ff1a537170224fdb853ac49e4e6a5b16, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c443665f26f24d028d41a4e068b9a457, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6981362edf644397aa193203b6d92a0c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b25a99571620456e8c22cf5cf27d8cc4, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c0672ad649c44a76a30875c538154c09, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/5f68a5efc0024beba33e8b48abde1fe8, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/bd3328d2e57f43a88b06cef0d649a14c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c65ede6ee42344fab06dd197423e5917, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b7614fa92d414d879923e3da4856c7ac, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6b0770f4f4aa4986be2d55f7e155d71d, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/336d4b28f67f45d299b6d8bc4f5217f5, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/dc7dbcd9dca142a9aba32b8f00a61d51, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/91fe5a22a51d4dd2b305a735c44e07b2, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/04ff877040164007adb1d064932dab8c, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/fdb2954a1784424dac40d1c4901d7615] to archive 2024-11-13T10:41:05,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T10:41:05,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/101f86312add43e79d07b5fab3181724 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/101f86312add43e79d07b5fab3181724 2024-11-13T10:41:05,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/d6d45f7d799e4bccaf812f24c522be58 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/d6d45f7d799e4bccaf812f24c522be58 2024-11-13T10:41:05,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/3ecc0874c1284833b53fa9378b9f476a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/3ecc0874c1284833b53fa9378b9f476a 2024-11-13T10:41:05,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/e83b736795b5407689af646a48fd39be to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/e83b736795b5407689af646a48fd39be 2024-11-13T10:41:05,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/97ace567baf644cabb91e5fc52df2345 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/97ace567baf644cabb91e5fc52df2345 2024-11-13T10:41:05,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/ff1a537170224fdb853ac49e4e6a5b16 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/ff1a537170224fdb853ac49e4e6a5b16 2024-11-13T10:41:05,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c443665f26f24d028d41a4e068b9a457 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c443665f26f24d028d41a4e068b9a457 2024-11-13T10:41:05,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6981362edf644397aa193203b6d92a0c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6981362edf644397aa193203b6d92a0c 2024-11-13T10:41:05,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b25a99571620456e8c22cf5cf27d8cc4 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b25a99571620456e8c22cf5cf27d8cc4 2024-11-13T10:41:05,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c0672ad649c44a76a30875c538154c09 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c0672ad649c44a76a30875c538154c09 2024-11-13T10:41:05,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/5f68a5efc0024beba33e8b48abde1fe8 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/5f68a5efc0024beba33e8b48abde1fe8 2024-11-13T10:41:05,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/bd3328d2e57f43a88b06cef0d649a14c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/bd3328d2e57f43a88b06cef0d649a14c 2024-11-13T10:41:05,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c65ede6ee42344fab06dd197423e5917 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/c65ede6ee42344fab06dd197423e5917 2024-11-13T10:41:05,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b7614fa92d414d879923e3da4856c7ac to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/b7614fa92d414d879923e3da4856c7ac 2024-11-13T10:41:05,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6b0770f4f4aa4986be2d55f7e155d71d to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/6b0770f4f4aa4986be2d55f7e155d71d 2024-11-13T10:41:05,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/336d4b28f67f45d299b6d8bc4f5217f5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/336d4b28f67f45d299b6d8bc4f5217f5 2024-11-13T10:41:05,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/dc7dbcd9dca142a9aba32b8f00a61d51 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/dc7dbcd9dca142a9aba32b8f00a61d51 2024-11-13T10:41:05,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/91fe5a22a51d4dd2b305a735c44e07b2 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/91fe5a22a51d4dd2b305a735c44e07b2 2024-11-13T10:41:05,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/04ff877040164007adb1d064932dab8c to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/04ff877040164007adb1d064932dab8c 2024-11-13T10:41:05,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/fdb2954a1784424dac40d1c4901d7615 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/fdb2954a1784424dac40d1c4901d7615 2024-11-13T10:41:05,368 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/recovered.edits/319.seqid, newMaxSeqId=319, maxSeqId=4 2024-11-13T10:41:05,368 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. 2024-11-13T10:41:05,368 DEBUG [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for c32615157847c6ad00128cef38e49033: Waiting for close lock at 1731494463614Running coprocessor pre-close hooks at 1731494463614Disabling compacts and flushes for region at 1731494463614Disabling writes for close at 1731494463614Obtaining lock to block concurrent updates at 1731494463614Preparing flush snapshotting stores in c32615157847c6ad00128cef38e49033 at 1731494463614Finished memstore snapshotting TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1731494463615 (+1 ms)Flushing stores of TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033. at 1731494463615Flushing c32615157847c6ad00128cef38e49033/A: creating writer at 1731494463615Flushing mob file c32615157847c6ad00128cef38e49033/A: appending metadata at 1731494463620 (+5 ms)Flushing mob file c32615157847c6ad00128cef38e49033/A: closing flushed file at 1731494463620Flushing c32615157847c6ad00128cef38e49033/A: appending metadata at 1731494464035 (+415 ms)Flushing c32615157847c6ad00128cef38e49033/A: closing flushed file at 1731494464036 (+1 ms)Flushing c32615157847c6ad00128cef38e49033/B: creating writer at 1731494464466 (+430 ms)Flushing c32615157847c6ad00128cef38e49033/B: appending metadata at 1731494464468 (+2 ms)Flushing c32615157847c6ad00128cef38e49033/B: closing flushed file at 1731494464468Flushing c32615157847c6ad00128cef38e49033/C: creating writer at 1731494464879 (+411 ms)Flushing c32615157847c6ad00128cef38e49033/C: appending metadata at 1731494464881 (+2 ms)Flushing c32615157847c6ad00128cef38e49033/C: closing flushed file at 1731494464881Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5015fec1: reopening flushed file at 1731494465296 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@727fe92d: reopening flushed file at 1731494465299 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a2f915d: reopening flushed file at 1731494465302 (+3 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for c32615157847c6ad00128cef38e49033 in 1691ms, sequenceid=316, compaction requested=true at 1731494465305 (+3 ms)Writing region close event to WAL at 1731494465365 (+60 ms)Running coprocessor post-close hooks at 1731494465368 (+3 ms)Closed at 1731494465368 2024-11-13T10:41:05,370 INFO [RS_CLOSE_REGION-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,370 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=c32615157847c6ad00128cef38e49033, regionState=CLOSED 2024-11-13T10:41:05,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 because future has completed 2024-11-13T10:41:05,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-13T10:41:05,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure c32615157847c6ad00128cef38e49033, server=770665a7984d,39101,1731494292956 in 1.9100 sec 2024-11-13T10:41:05,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-13T10:41:05,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c32615157847c6ad00128cef38e49033, UNASSIGN in 1.9150 sec 2024-11-13T10:41:05,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-13T10:41:05,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9190 sec 2024-11-13T10:41:05,377 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731494465377"}]},"ts":"1731494465377"} 2024-11-13T10:41:05,378 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T10:41:05,378 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T10:41:05,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9300 sec 2024-11-13T10:41:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-13T10:41:05,597 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:41:05,598 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T10:41:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:41:05,600 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:41:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-13T10:41:05,600 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:41:05,603 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,605 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C, FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/recovered.edits] 2024-11-13T10:41:05,608 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6f2dc8ca7e824a6bb46d0c93377dbca5 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/6f2dc8ca7e824a6bb46d0c93377dbca5 2024-11-13T10:41:05,609 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/ac0041c02cea44afa318bdca656e3830 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/ac0041c02cea44afa318bdca656e3830 2024-11-13T10:41:05,610 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/c9556a871d00474492cc72ef52a51a1a to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/c9556a871d00474492cc72ef52a51a1a 2024-11-13T10:41:05,613 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f5df1ab51f3148a38b694e8dbadeb4ca to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/A/f5df1ab51f3148a38b694e8dbadeb4ca 2024-11-13T10:41:05,616 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/3127f57e9eb64101a54ef555ac74a375 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/3127f57e9eb64101a54ef555ac74a375 2024-11-13T10:41:05,617 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/6aaa72ad7f3745a1a7962c095b893623 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/6aaa72ad7f3745a1a7962c095b893623 2024-11-13T10:41:05,618 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b58b0d20888649e791cd08fb8e759487 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b58b0d20888649e791cd08fb8e759487 2024-11-13T10:41:05,619 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b824704b5429435c9ab3b0ddd2c243df to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/B/b824704b5429435c9ab3b0ddd2c243df 2024-11-13T10:41:05,621 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/536df1949ec14a98b23df2bf8509a304 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/536df1949ec14a98b23df2bf8509a304 2024-11-13T10:41:05,621 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/62cc32a1693e42569d8f6b423bd982f0 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/62cc32a1693e42569d8f6b423bd982f0 2024-11-13T10:41:05,622 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/8d2dcfbf602e4e7baf61772e65095b19 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/8d2dcfbf602e4e7baf61772e65095b19 2024-11-13T10:41:05,623 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/9073ccfd8eff495692ce3b16c9ec165e to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/C/9073ccfd8eff495692ce3b16c9ec165e 2024-11-13T10:41:05,624 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/recovered.edits/319.seqid to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033/recovered.edits/319.seqid 2024-11-13T10:41:05,625 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/default/TestAcidGuarantees/c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,625 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T10:41:05,625 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T10:41:05,626 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T10:41:05,628 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308f6f9a4927c45eea76b2b0f76e7c6ed_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111308f6f9a4927c45eea76b2b0f76e7c6ed_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,629 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130ad1e3abc7824ea4bd0fcab5f6a7c39e_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130ad1e3abc7824ea4bd0fcab5f6a7c39e_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,630 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130b0db71dbe064fe39b6f8ea5922e0095_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130b0db71dbe064fe39b6f8ea5922e0095_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,631 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130cb091fa10ac4ecf9973ca6e6ceed243_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130cb091fa10ac4ecf9973ca6e6ceed243_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,632 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111314714183bc9143198c4fa3a0ea408c8d_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111314714183bc9143198c4fa3a0ea408c8d_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,633 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111318e198435e8644ffafaa499034506ec7_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111318e198435e8644ffafaa499034506ec7_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,634 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111337991e6bf5b149319336e0e99ed1ab15_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111337991e6bf5b149319336e0e99ed1ab15_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,635 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111338497f674c28401d99afee813b6ac26e_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111338497f674c28401d99afee813b6ac26e_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,636 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111352b7ac2f1c8e49b6bc378a36e11c114c_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111352b7ac2f1c8e49b6bc378a36e11c114c_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,637 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136c9298ee457a4147baf71a0c1a9ba6da_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136c9298ee457a4147baf71a0c1a9ba6da_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,638 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136d380e70d64248a790cff8e594b21469_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136d380e70d64248a790cff8e594b21469_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,639 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111384c80b3e69da4d4698250c145b2f72b2_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111384c80b3e69da4d4698250c145b2f72b2_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,640 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138907141e621149b69af33e69f8101f50_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138907141e621149b69af33e69f8101f50_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,641 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c59db86178514109be6865adc3f82a7b_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c59db86178514109be6865adc3f82a7b_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,641 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c8e41cb508fc4e7484aed3a313635df4_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c8e41cb508fc4e7484aed3a313635df4_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,642 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e5188aa3084e436cbdd45200b456b0d5_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e5188aa3084e436cbdd45200b456b0d5_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,643 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ed7a8637ce9e4028b9fa89e5815385ed_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ed7a8637ce9e4028b9fa89e5815385ed_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,644 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f7a7327fd3274155bd07b3354df8a560_c32615157847c6ad00128cef38e49033 to hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f7a7327fd3274155bd07b3354df8a560_c32615157847c6ad00128cef38e49033 2024-11-13T10:41:05,644 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T10:41:05,646 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:41:05,648 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T10:41:05,649 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T10:41:05,650 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:41:05,650 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T10:41:05,650 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731494465650"}]},"ts":"9223372036854775807"} 2024-11-13T10:41:05,652 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T10:41:05,652 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c32615157847c6ad00128cef38e49033, NAME => 'TestAcidGuarantees,,1731494437992.c32615157847c6ad00128cef38e49033.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T10:41:05,652 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T10:41:05,652 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731494465652"}]},"ts":"9223372036854775807"} 2024-11-13T10:41:05,653 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T10:41:05,654 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T10:41:05,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-13T10:41:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-13T10:41:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T10:41:05,707 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T10:41:05,719 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 240) - Thread LEAK? -, OpenFileDescriptor=459 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=838 (was 768) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=816 (was 514) - AvailableMemoryMB LEAK? - 2024-11-13T10:41:05,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T10:41:05,719 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T10:41:05,719 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T10:41:05,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:05,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:05,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T10:41:05,720 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T10:41:05,720 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1372924334, stopped=false 2024-11-13T10:41:05,720 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=770665a7984d,44523,1731494292131 2024-11-13T10:41:05,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T10:41:05,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T10:41:05,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:41:05,725 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T10:41:05,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:41:05,725 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T10:41:05,725 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T10:41:05,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:05,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T10:41:05,726 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '770665a7984d,39101,1731494292956' ***** 2024-11-13T10:41:05,726 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T10:41:05,726 INFO [RS:0;770665a7984d:39101 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T10:41:05,727 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T10:41:05,727 INFO [RS:0;770665a7984d:39101 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T10:41:05,727 INFO [RS:0;770665a7984d:39101 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T10:41:05,727 INFO [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(959): stopping server 770665a7984d,39101,1731494292956 2024-11-13T10:41:05,727 INFO [RS:0;770665a7984d:39101 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T10:41:05,727 INFO [RS:0;770665a7984d:39101 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;770665a7984d:39101. 2024-11-13T10:41:05,727 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T10:41:05,727 DEBUG [RS:0;770665a7984d:39101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T10:41:05,727 DEBUG [RS:0;770665a7984d:39101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:05,728 INFO [RS:0;770665a7984d:39101 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T10:41:05,728 INFO [RS:0;770665a7984d:39101 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T10:41:05,728 INFO [RS:0;770665a7984d:39101 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T10:41:05,728 INFO [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T10:41:05,729 INFO [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T10:41:05,729 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T10:41:05,729 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T10:41:05,729 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T10:41:05,729 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T10:41:05,729 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T10:41:05,729 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T10:41:05,729 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T10:41:05,729 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-13T10:41:05,750 INFO [regionserver/770665a7984d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T10:41:05,758 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/info/9772a24b768342089b2ddeebe663cf12 is 95, key is TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9./info:/1731494323039/DeleteFamily/seqid=0 2024-11-13T10:41:05,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742490_1666 (size=6514) 2024-11-13T10:41:05,769 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/info/9772a24b768342089b2ddeebe663cf12 2024-11-13T10:41:05,794 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/ns/4f4b8725de964d73b46cfaf2d23d1065 is 93, key is TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9./ns:/1731494322992/DeleteFamily/seqid=0 2024-11-13T10:41:05,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742491_1667 (size=5906) 2024-11-13T10:41:05,929 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T10:41:06,129 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T10:41:06,200 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/ns/4f4b8725de964d73b46cfaf2d23d1065 2024-11-13T10:41:06,228 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/rep_barrier/4cc76aec4fef49368a20f84c2561c108 is 102, key is TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9./rep_barrier:/1731494322992/DeleteFamily/seqid=0 2024-11-13T10:41:06,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742492_1668 (size=6025) 2024-11-13T10:41:06,244 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/rep_barrier/4cc76aec4fef49368a20f84c2561c108 2024-11-13T10:41:06,269 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/table/289eba15a818406a885b5316b0c611cb is 96, key is TestAcidGuarantees,,1731494297549.0704f3508555649d4fe7145badb2caf9./table:/1731494322992/DeleteFamily/seqid=0 2024-11-13T10:41:06,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742493_1669 (size=5936) 2024-11-13T10:41:06,330 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T10:41:06,360 INFO [regionserver/770665a7984d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T10:41:06,360 INFO [regionserver/770665a7984d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T10:41:06,530 DEBUG [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T10:41:06,682 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/table/289eba15a818406a885b5316b0c611cb 2024-11-13T10:41:06,687 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/info/9772a24b768342089b2ddeebe663cf12 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/info/9772a24b768342089b2ddeebe663cf12 2024-11-13T10:41:06,691 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/info/9772a24b768342089b2ddeebe663cf12, entries=12, sequenceid=90, filesize=6.4 K 2024-11-13T10:41:06,692 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/ns/4f4b8725de964d73b46cfaf2d23d1065 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/ns/4f4b8725de964d73b46cfaf2d23d1065 2024-11-13T10:41:06,695 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/ns/4f4b8725de964d73b46cfaf2d23d1065, entries=8, sequenceid=90, filesize=5.8 K 2024-11-13T10:41:06,696 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/rep_barrier/4cc76aec4fef49368a20f84c2561c108 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/rep_barrier/4cc76aec4fef49368a20f84c2561c108 2024-11-13T10:41:06,699 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/rep_barrier/4cc76aec4fef49368a20f84c2561c108, entries=6, sequenceid=90, filesize=5.9 K 2024-11-13T10:41:06,700 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/.tmp/table/289eba15a818406a885b5316b0c611cb as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/table/289eba15a818406a885b5316b0c611cb 2024-11-13T10:41:06,703 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/table/289eba15a818406a885b5316b0c611cb, entries=7, sequenceid=90, filesize=5.8 K 2024-11-13T10:41:06,703 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 974ms, sequenceid=90, compaction requested=false 2024-11-13T10:41:06,719 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-13T10:41:06,721 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T10:41:06,721 INFO [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T10:41:06,721 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731494465729Running coprocessor pre-close hooks at 1731494465729Disabling compacts and flushes for region at 1731494465729Disabling writes for close at 1731494465729Obtaining lock to block concurrent updates at 1731494465729Preparing flush snapshotting stores in 1588230740 at 1731494465729Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731494465729Flushing stores of hbase:meta,,1.1588230740 at 1731494465730 (+1 ms)Flushing 1588230740/info: creating writer at 1731494465730Flushing 1588230740/info: appending metadata at 1731494465758 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731494465758Flushing 1588230740/ns: creating writer at 1731494465772 (+14 ms)Flushing 1588230740/ns: appending metadata at 1731494465793 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731494465793Flushing 1588230740/rep_barrier: creating writer at 1731494466204 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731494466228 (+24 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731494466228Flushing 1588230740/table: creating writer at 1731494466248 (+20 ms)Flushing 1588230740/table: appending metadata at 1731494466269 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731494466269Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c2c8f7: reopening flushed file at 1731494466687 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cf0879d: reopening flushed file at 1731494466691 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47a929b7: reopening flushed file at 1731494466696 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@291e7240: reopening flushed file at 1731494466700 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 974ms, sequenceid=90, compaction requested=false at 1731494466703 (+3 ms)Writing region close event to WAL at 1731494466716 (+13 ms)Running coprocessor post-close hooks at 1731494466721 (+5 ms)Closed at 1731494466721 2024-11-13T10:41:06,721 DEBUG [RS_CLOSE_META-regionserver/770665a7984d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T10:41:06,730 INFO [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(976): stopping server 770665a7984d,39101,1731494292956; all regions closed. 2024-11-13T10:41:06,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741834_1010 (size=25344) 2024-11-13T10:41:06,740 DEBUG [RS:0;770665a7984d:39101 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/oldWALs 2024-11-13T10:41:06,741 INFO [RS:0;770665a7984d:39101 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 770665a7984d%2C39101%2C1731494292956.meta:.meta(num 1731494297036) 2024-11-13T10:41:06,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741833_1009 (size=15586972) 2024-11-13T10:41:06,749 DEBUG [RS:0;770665a7984d:39101 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/oldWALs 2024-11-13T10:41:06,749 INFO [RS:0;770665a7984d:39101 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 770665a7984d%2C39101%2C1731494292956:(num 1731494296582) 2024-11-13T10:41:06,749 DEBUG [RS:0;770665a7984d:39101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T10:41:06,749 INFO [RS:0;770665a7984d:39101 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T10:41:06,749 INFO [RS:0;770665a7984d:39101 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T10:41:06,749 INFO [RS:0;770665a7984d:39101 {}] hbase.ChoreService(370): Chore service for: regionserver/770665a7984d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T10:41:06,749 INFO [RS:0;770665a7984d:39101 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T10:41:06,750 INFO [RS:0;770665a7984d:39101 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39101 2024-11-13T10:41:06,753 INFO [regionserver/770665a7984d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T10:41:06,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/770665a7984d,39101,1731494292956 2024-11-13T10:41:06,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T10:41:06,755 INFO [RS:0;770665a7984d:39101 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T10:41:06,755 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [770665a7984d,39101,1731494292956] 2024-11-13T10:41:06,760 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/770665a7984d,39101,1731494292956 already deleted, retry=false 2024-11-13T10:41:06,761 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 770665a7984d,39101,1731494292956 expired; onlineServers=0 2024-11-13T10:41:06,761 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '770665a7984d,44523,1731494292131' ***** 2024-11-13T10:41:06,761 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T10:41:06,761 INFO [M:0;770665a7984d:44523 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T10:41:06,761 INFO [M:0;770665a7984d:44523 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T10:41:06,761 DEBUG [M:0;770665a7984d:44523 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T10:41:06,761 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T10:41:06,761 DEBUG [M:0;770665a7984d:44523 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T10:41:06,761 DEBUG [master/770665a7984d:0:becomeActiveMaster-HFileCleaner.small.0-1731494295961 {}] cleaner.HFileCleaner(306): Exit Thread[master/770665a7984d:0:becomeActiveMaster-HFileCleaner.small.0-1731494295961,5,FailOnTimeoutGroup] 2024-11-13T10:41:06,761 DEBUG [master/770665a7984d:0:becomeActiveMaster-HFileCleaner.large.0-1731494295939 {}] cleaner.HFileCleaner(306): Exit Thread[master/770665a7984d:0:becomeActiveMaster-HFileCleaner.large.0-1731494295939,5,FailOnTimeoutGroup] 2024-11-13T10:41:06,762 INFO [M:0;770665a7984d:44523 {}] hbase.ChoreService(370): Chore service for: master/770665a7984d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T10:41:06,762 INFO [M:0;770665a7984d:44523 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T10:41:06,762 DEBUG [M:0;770665a7984d:44523 {}] master.HMaster(1795): Stopping service threads 2024-11-13T10:41:06,762 INFO [M:0;770665a7984d:44523 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T10:41:06,762 INFO [M:0;770665a7984d:44523 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T10:41:06,762 ERROR [M:0;770665a7984d:44523 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (2068722025) connection to localhost/127.0.0.1:42945 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:42945,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-13T10:41:06,763 INFO [M:0;770665a7984d:44523 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T10:41:06,763 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T10:41:06,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T10:41:06,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T10:41:06,765 DEBUG [M:0;770665a7984d:44523 {}] zookeeper.ZKUtil(347): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T10:41:06,765 WARN [M:0;770665a7984d:44523 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T10:41:06,765 INFO [M:0;770665a7984d:44523 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/.lastflushedseqids 2024-11-13T10:41:06,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742494_1670 (size=72) 2024-11-13T10:41:06,791 INFO [M:0;770665a7984d:44523 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T10:41:06,791 INFO [M:0;770665a7984d:44523 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T10:41:06,791 DEBUG [M:0;770665a7984d:44523 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T10:41:06,791 INFO [M:0;770665a7984d:44523 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T10:41:06,791 DEBUG [M:0;770665a7984d:44523 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T10:41:06,791 DEBUG [M:0;770665a7984d:44523 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T10:41:06,791 DEBUG [M:0;770665a7984d:44523 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T10:41:06,791 INFO [M:0;770665a7984d:44523 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=771.86 KB heapSize=949.88 KB 2024-11-13T10:41:06,810 DEBUG [M:0;770665a7984d:44523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c13de02d73444f9bc5b06af195d3d83 is 82, key is hbase:meta,,1/info:regioninfo/1731494297143/Put/seqid=0 2024-11-13T10:41:06,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742495_1671 (size=5672) 2024-11-13T10:41:06,829 INFO [M:0;770665a7984d:44523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2201 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c13de02d73444f9bc5b06af195d3d83 2024-11-13T10:41:06,858 INFO [RS:0;770665a7984d:39101 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T10:41:06,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T10:41:06,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39101-0x10110e811380001, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T10:41:06,859 INFO [RS:0;770665a7984d:39101 {}] regionserver.HRegionServer(1031): Exiting; stopping=770665a7984d,39101,1731494292956; zookeeper connection closed. 2024-11-13T10:41:06,860 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@257c2c0c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@257c2c0c 2024-11-13T10:41:06,861 DEBUG [M:0;770665a7984d:44523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0bdf88966f624849af8afe95dc702548 is 2275, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1731494441155/Put/seqid=0 2024-11-13T10:41:06,861 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T10:41:06,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742496_1672 (size=44477) 2024-11-13T10:41:06,864 INFO [M:0;770665a7984d:44523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=771.30 KB at sequenceid=2201 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0bdf88966f624849af8afe95dc702548 2024-11-13T10:41:06,868 INFO [M:0;770665a7984d:44523 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0bdf88966f624849af8afe95dc702548 2024-11-13T10:41:06,892 DEBUG [M:0;770665a7984d:44523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/455514c1d80646aa9cd1163488c4da72 is 69, key is 770665a7984d,39101,1731494292956/rs:state/1731494296111/Put/seqid=0 2024-11-13T10:41:06,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073742497_1673 (size=5156) 2024-11-13T10:41:07,329 INFO [M:0;770665a7984d:44523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2201 (bloomFilter=true), to=hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/455514c1d80646aa9cd1163488c4da72 2024-11-13T10:41:07,333 DEBUG [M:0;770665a7984d:44523 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c13de02d73444f9bc5b06af195d3d83 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8c13de02d73444f9bc5b06af195d3d83 2024-11-13T10:41:07,337 INFO [M:0;770665a7984d:44523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8c13de02d73444f9bc5b06af195d3d83, entries=8, sequenceid=2201, filesize=5.5 K 2024-11-13T10:41:07,337 DEBUG [M:0;770665a7984d:44523 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0bdf88966f624849af8afe95dc702548 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0bdf88966f624849af8afe95dc702548 2024-11-13T10:41:07,340 INFO [M:0;770665a7984d:44523 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0bdf88966f624849af8afe95dc702548 2024-11-13T10:41:07,340 INFO [M:0;770665a7984d:44523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0bdf88966f624849af8afe95dc702548, entries=172, sequenceid=2201, filesize=43.4 K 2024-11-13T10:41:07,341 DEBUG [M:0;770665a7984d:44523 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/455514c1d80646aa9cd1163488c4da72 as hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/455514c1d80646aa9cd1163488c4da72 2024-11-13T10:41:07,344 INFO [M:0;770665a7984d:44523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42945/user/jenkins/test-data/8c5c1b0d-7edb-5e8d-34b4-18ca2bbff617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/455514c1d80646aa9cd1163488c4da72, entries=1, sequenceid=2201, filesize=5.0 K 2024-11-13T10:41:07,345 INFO [M:0;770665a7984d:44523 {}] regionserver.HRegion(3140): Finished flush of dataSize ~771.86 KB/790385, heapSize ~949.58 KB/972368, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 554ms, sequenceid=2201, compaction requested=false 2024-11-13T10:41:07,347 INFO [M:0;770665a7984d:44523 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T10:41:07,347 DEBUG [M:0;770665a7984d:44523 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731494466791Disabling compacts and flushes for region at 1731494466791Disabling writes for close at 1731494466791Obtaining lock to block concurrent updates at 1731494466791Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731494466791Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=790385, getHeapSize=972608, getOffHeapSize=0, getCellsCount=2398 at 1731494466791Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731494466792 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731494466792Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731494466810 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731494466810Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731494466833 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731494466860 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731494466860Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731494466868 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731494466892 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731494466892Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36292f89: reopening flushed file at 1731494467332 (+440 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@662ec6cd: reopening flushed file at 1731494467337 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33c508fa: reopening flushed file at 1731494467340 (+3 ms)Finished flush of dataSize ~771.86 KB/790385, heapSize ~949.58 KB/972368, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 554ms, sequenceid=2201, compaction requested=false at 1731494467345 (+5 ms)Writing region close event to WAL at 1731494467347 (+2 ms)Closed at 1731494467347 2024-11-13T10:41:07,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40603 is added to blk_1073741830_1006 (size=934657) 2024-11-13T10:41:07,351 INFO [M:0;770665a7984d:44523 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T10:41:07,352 INFO [M:0;770665a7984d:44523 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44523 2024-11-13T10:41:07,352 INFO [M:0;770665a7984d:44523 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T10:41:07,352 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T10:41:07,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T10:41:07,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44523-0x10110e811380000, quorum=127.0.0.1:49884, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T10:41:07,455 INFO [M:0;770665a7984d:44523 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T10:41:07,476 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T10:41:07,480 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T10:41:07,481 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T10:41:07,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T10:41:07,482 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/hadoop.log.dir/,STOPPED} 2024-11-13T10:41:07,490 WARN [BP-1227865098-172.17.0.2-1731494288673 heartbeating to localhost/127.0.0.1:42945 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T10:41:07,490 WARN [BP-1227865098-172.17.0.2-1731494288673 heartbeating to localhost/127.0.0.1:42945 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1227865098-172.17.0.2-1731494288673 (Datanode Uuid 31cb90c5-5f29-4598-9974-72160ff757af) service to localhost/127.0.0.1:42945 2024-11-13T10:41:07,494 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T10:41:07,494 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T10:41:07,494 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/cluster_deb536b6-0585-3c7e-f24a-dacaede12fac/data/data1/current/BP-1227865098-172.17.0.2-1731494288673 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T10:41:07,494 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/cluster_deb536b6-0585-3c7e-f24a-dacaede12fac/data/data2/current/BP-1227865098-172.17.0.2-1731494288673 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T10:41:07,496 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T10:41:07,513 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T10:41:07,519 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T10:41:07,519 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T10:41:07,520 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T10:41:07,520 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/798ddde4-e79b-b15d-eb37-8c490c5d64ee/hadoop.log.dir/,STOPPED} 2024-11-13T10:41:07,551 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T10:41:07,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down